diff --git a/common/web/input-processor/tests/cases/languageProcessor.js b/common/web/input-processor/tests/cases/languageProcessor.js index 7391624004..5f7a92adc3 100644 --- a/common/web/input-processor/tests/cases/languageProcessor.js +++ b/common/web/input-processor/tests/cases/languageProcessor.js @@ -57,7 +57,13 @@ describe('LanguageProcessor', function() { }); describe('.predict', function() { - let compiler = new LexicalModelCompiler(callbacks); + let compiler = null; + + this.beforeAll(async function() { + compiler = new LexicalModelCompiler(); + assert.isTrue(await compiler.init(callbacks, {})); + }); + const MODEL_ID = 'example.qaa.trivial'; // ES-module mode leaves out `__dirname`, so we rebuild it using other components. @@ -67,20 +73,23 @@ describe('LanguageProcessor', function() { const PATH = path.join(__dirname, '../../../../../developer/src/kmc-model/test/fixtures', MODEL_ID); describe('using angle brackets for quotes', function() { - let modelCode = compiler.generateLexicalModelCode(MODEL_ID, { - format: 'trie-1.0', - sources: ['wordlist.tsv'], - punctuation: { - quotesForKeepSuggestion: { open: `«`, close: `»`}, - insertAfterWord: " " , // OGHAM SPACE MARK - } - }, PATH); + let modelCode = null, modelSpec = null; + this.beforeAll(function() { + modelCode = compiler.generateLexicalModelCode(MODEL_ID, { + format: 'trie-1.0', + sources: ['wordlist.tsv'], + punctuation: { + quotesForKeepSuggestion: { open: `«`, close: `»`}, + insertAfterWord: " " , // OGHAM SPACE MARK + } + }, PATH); - let modelSpec = { - id: MODEL_ID, - languages: ['en'], - code: modelCode - }; + modelSpec = { + id: MODEL_ID, + languages: ['en'], + code: modelCode + }; + }); it("successfully loads the model", function(done) { let languageProcessor = new LanguageProcessor(worker, new TranscriptionCache()); @@ -115,18 +124,21 @@ describe('LanguageProcessor', function() { }); describe('properly cases generated suggestions', function() { - let modelCode = compiler.generateLexicalModelCode(MODEL_ID, { - format: 'trie-1.0', - sources: ['wordlist.tsv'], - languageUsesCasing: true, - //applyCasing // we rely on the compiler's default implementation here. - }, PATH); + let modelCode = null, modelSpec = null; + this.beforeAll(function () { + modelCode = compiler.generateLexicalModelCode(MODEL_ID, { + format: 'trie-1.0', + sources: ['wordlist.tsv'], + languageUsesCasing: true, + //applyCasing // we rely on the compiler's default implementation here. + }, PATH); - let modelSpec = { - id: MODEL_ID, - languages: ['en'], - code: modelCode - }; + modelSpec = { + id: MODEL_ID, + languages: ['en'], + code: modelCode + }; + }); describe("does not alter casing when input is lowercased", function() { it("when input is fully lowercased", function(done) { diff --git a/common/web/types/src/kpj/keyman-developer-project.ts b/common/web/types/src/kpj/keyman-developer-project.ts index a53cbca20e..fdc8466497 100644 --- a/common/web/types/src/kpj/keyman-developer-project.ts +++ b/common/web/types/src/kpj/keyman-developer-project.ts @@ -37,8 +37,16 @@ export class KeymanDeveloperProject { for(let filename of files) { let fullPath = this.callbacks.path.join(sourcePath, filename); if(KeymanFileTypes.filenameIs(filename, KeymanFileTypes.Source.LdmlKeyboard)) { - if(!this.callbacks.fs.readFileSync(fullPath, 'utf-8').match(/ldmlKeyboard3\.dtd/)) { - // Skip this .xml because we assume it isn't really a keyboard .xml + try { + const data = this.callbacks.loadFile(fullPath); + const text = new TextDecoder().decode(data); + if(!text?.match(/ldmlKeyboard3\.dtd/)) { + // Skip this .xml because we assume it isn't really a keyboard .xml + continue; + } + } catch(e) { + // We'll just silently skip this file because we were not able to load it, + // so let's hope it wasn't a real LDML keyboard XML :-) continue; } } diff --git a/common/web/types/src/main.ts b/common/web/types/src/main.ts index 06671a13d7..4a1afc1bf9 100644 --- a/common/web/types/src/main.ts +++ b/common/web/types/src/main.ts @@ -27,6 +27,13 @@ export { defaultCompilerOptions, CompilerBaseOptions, CompilerCallbacks, Compile compilerExceptionToString, compilerErrorFormatCode, compilerLogLevelToSeverity, CompilerLogLevel, compilerEventFormat, ALL_COMPILER_LOG_LEVELS, ALL_COMPILER_LOG_FORMATS, CompilerLogFormat, + + KeymanCompilerArtifact, + KeymanCompilerArtifactOptional, + KeymanCompilerArtifacts, + KeymanCompilerResult, + KeymanCompiler + } from './util/compiler-interfaces.js'; export { CommonTypesMessages } from './util/common-events.js'; diff --git a/common/web/types/src/util/compiler-interfaces.ts b/common/web/types/src/util/compiler-interfaces.ts index b27a170ab3..6409c934f5 100644 --- a/common/web/types/src/util/compiler-interfaces.ts +++ b/common/web/types/src/util/compiler-interfaces.ts @@ -253,6 +253,40 @@ export interface CompilerCallbackOptions { compilerWarningsAsErrors?: boolean; }; +export interface KeymanCompilerArtifact { + data: Uint8Array; + filename: string; +}; + +export type KeymanCompilerArtifactOptional = KeymanCompilerArtifact | undefined; + +export interface KeymanCompilerArtifacts { + readonly [type:string]: KeymanCompilerArtifactOptional; +}; + +export interface KeymanCompilerResult { + artifacts: KeymanCompilerArtifacts; +}; + +export interface KeymanCompiler { + init(callbacks: CompilerCallbacks, options: CompilerOptions): Promise; + /** + * Run the compiler, and save the result in memory arrays. Note that while + * `outputFilename` is provided here, the output file is not written to in + * this function. + * @param inputFilename + * @param outputFilename The intended output filename, optional, if missing, + * calculated from inputFilename + * @param data + */ + run(inputFilename:string, outputFilename?:string /*, data?: any*/): Promise; + /** + * Writes the compiled output files to disk + * @param artifacts + */ + write(artifacts: KeymanCompilerArtifacts): Promise; +}; + /** * Abstract interface for callbacks, to abstract out file i/o */ @@ -369,10 +403,6 @@ export interface CompilerBaseOptions { * Format of output for log to console */ logFormat?: CompilerLogFormat; - /** - * Optional output file for activities that generate output - */ - outFile?: string; /** * Colorize log output, default is detected from console */ diff --git a/developer/src/kmc-analyze/src/util/get-osk-from-kmn-file.ts b/developer/src/kmc-analyze/src/util/get-osk-from-kmn-file.ts index 2df34e9dae..0e2ec28995 100644 --- a/developer/src/kmc-analyze/src/util/get-osk-from-kmn-file.ts +++ b/developer/src/kmc-analyze/src/util/get-osk-from-kmn-file.ts @@ -9,15 +9,15 @@ export async function getOskFromKmnFile(callbacks: CompilerCallbacks, filename: let touchLayoutFilename: string; const kmnCompiler = new KmnCompiler(); - if(!await kmnCompiler.init(callbacks)) { + if(!await kmnCompiler.init(callbacks, { + shouldAddCompilerVersion: false, + saveDebug: false, + })) { // kmnCompiler will report errors return null; } - let result = kmnCompiler.runCompiler(filename, { - shouldAddCompilerVersion: false, - saveDebug: false, - }); + let result = await kmnCompiler.run(filename, null); if(!result) { // kmnCompiler will report any errors @@ -29,7 +29,7 @@ export async function getOskFromKmnFile(callbacks: CompilerCallbacks, filename: } const reader = new KmxFileReader(); - const keyboard: KMX.KEYBOARD = reader.read(result.kmx.data); + const keyboard: KMX.KEYBOARD = reader.read(result.artifacts.kmx.data); const touchLayoutStore = keyboard.stores.find(store => store.dwSystemID == KMX.KMXFile.TSS_LAYOUTFILE); if(touchLayoutStore) { diff --git a/developer/src/kmc-keyboard-info/src/index.ts b/developer/src/kmc-keyboard-info/src/index.ts index beb4b88bb9..7ee9091610 100644 --- a/developer/src/kmc-keyboard-info/src/index.ts +++ b/developer/src/kmc-keyboard-info/src/index.ts @@ -5,7 +5,7 @@ import { minKeymanVersion } from "./min-keyman-version.js"; import { KeyboardInfoFile, KeyboardInfoFileIncludes, KeyboardInfoFileLanguageFont, KeyboardInfoFilePlatform } from "./keyboard-info-file.js"; -import { KeymanFileTypes, CompilerCallbacks, KmpJsonFile, KmxFileReader, KMX, KeymanTargets } from "@keymanapp/common-types"; +import { KeymanFileTypes, CompilerCallbacks, KmpJsonFile, KmxFileReader, KMX, KeymanTargets, KeymanCompiler, CompilerOptions, KeymanCompilerResult, KeymanCompilerArtifacts, KeymanCompilerArtifact } from "@keymanapp/common-types"; import { KeyboardInfoCompilerMessages } from "./messages.js"; import langtags from "./imports/langtags.js"; import { validateMITLicense } from "@keymanapp/developer-utils"; @@ -24,7 +24,7 @@ const HelpRoot = 'https://help.keyman.com/keyboard/'; * Build a dictionary of language tags from langtags.json */ -function init(): void { +function preinit(): void { if(langtagsByTag['en']) { // Already initialized, we can reasonably assume that 'en' will always be in // langtags.json. @@ -62,9 +62,30 @@ export interface KeyboardInfoSources { forPublishing: boolean; }; -export class KeyboardInfoCompiler { - constructor(private callbacks: CompilerCallbacks) { - init(); +export interface KeyboardInfoCompilerOptions extends CompilerOptions { + sources: KeyboardInfoSources; +}; + +export interface KeyboardInfoCompilerArtifacts extends KeymanCompilerArtifacts { + keyboard_info: KeymanCompilerArtifact; +}; + +export interface KeyboardInfoCompilerResult extends KeymanCompilerResult { + artifacts: KeyboardInfoCompilerArtifacts; +}; + +export class KeyboardInfoCompiler implements KeymanCompiler { + private callbacks: CompilerCallbacks; + private options: KeyboardInfoCompilerOptions; + + constructor() { + preinit(); + } + + public async init(callbacks: CompilerCallbacks, options: KeyboardInfoCompilerOptions): Promise { + this.callbacks = callbacks; + this.options = {...options}; + return true; } /** @@ -77,15 +98,18 @@ export class KeyboardInfoCompiler { * * @param sources Details on files from which to extract metadata */ - public async writeKeyboardInfoFile( - sources: KeyboardInfoSources - ): Promise { + public async run(inputFilename: string, outputFilename?: string): Promise { + const sources = this.options.sources; // TODO(lowpri): work from .kpj and nothing else as input. Blocked because // .kpj work is largely in kmc at present, so that would need to move to // a separate module. - const kmpCompiler = new KmpCompiler(this.callbacks); + const kmpCompiler = new KmpCompiler(); + if(!await kmpCompiler.init(this.callbacks, {})) { + // Errors will have been emitted by KmpCompiler + return null; + } const kmpJsonData = kmpCompiler.transformKpsToKmpObject(sources.kpsFilename); if(!kmpJsonData) { // Errors will have been emitted by KmpCompiler @@ -307,7 +331,22 @@ export class KeyboardInfoCompiler { }, null, 2)); } - return new TextEncoder().encode(jsonOutput); + const data = new TextEncoder().encode(jsonOutput); + const result: KeyboardInfoCompilerResult = { + artifacts: { + keyboard_info: { + data, + filename: outputFilename ?? inputFilename.replace(/\.kpj$/, '.keyboard_info') + } + } + }; + + return result; + } + + public async write(artifacts: KeyboardInfoCompilerArtifacts): Promise { + this.callbacks.fs.writeFileSync(artifacts.keyboard_info.filename, artifacts.keyboard_info.data); + return true; } private mapKeymanTargetToPlatform(target: KeymanTargets.KeymanTarget): KeyboardInfoFilePlatform[] { diff --git a/developer/src/kmc-keyboard-info/test/test-keyboard-info-compiler.ts b/developer/src/kmc-keyboard-info/test/test-keyboard-info-compiler.ts index 40ff764285..eab1202c8c 100644 --- a/developer/src/kmc-keyboard-info/test/test-keyboard-info-compiler.ts +++ b/developer/src/kmc-keyboard-info/test/test-keyboard-info-compiler.ts @@ -3,7 +3,7 @@ import { assert } from 'chai'; import 'mocha'; import { TestCompilerCallbacks } from '@keymanapp/developer-test-helpers'; import { makePathToFixture } from './helpers/index.js'; -import { KeyboardInfoCompiler } from '../src/index.js'; +import { KeyboardInfoCompiler, KeyboardInfoCompilerResult } from '../src/index.js'; const callbacks = new TestCompilerCallbacks(); @@ -13,31 +13,35 @@ beforeEach(function() { describe('keyboard-info-compiler', function () { it('compile a .keyboard_info file correctly', async function() { + const kpjFilename = makePathToFixture('khmer_angkor', 'khmer_angkor.kpj'); const jsFilename = makePathToFixture('khmer_angkor', 'build', 'khmer_angkor.js'); const kpsFilename = makePathToFixture('khmer_angkor', 'source', 'khmer_angkor.kps'); const kmpFilename = makePathToFixture('khmer_angkor', 'build', 'khmer_angkor.kmp'); const buildKeyboardInfoFilename = makePathToFixture('khmer_angkor', 'build', 'khmer_angkor.keyboard_info'); - const compiler = new KeyboardInfoCompiler(callbacks); - let data = null; + const sources = { + kmpFilename, + sourcePath: 'release/k/khmer_angkor', + kpsFilename, + jsFilename: jsFilename, + forPublishing: true, + }; + + const compiler = new KeyboardInfoCompiler(); + assert.isTrue(await compiler.init(callbacks, {sources})); + let result: KeyboardInfoCompilerResult = null; try { - data = await compiler.writeKeyboardInfoFile({ - kmpFilename, - sourcePath: 'release/k/khmer_angkor', - kpsFilename, - jsFilename: jsFilename, - forPublishing: true, - }); + result = await compiler.run(kpjFilename, null); } catch(e) { callbacks.printMessages(); throw e; } - if(data == null) { + if(result == null) { callbacks.printMessages(); } - assert.isNotNull(data); + assert.isNotNull(result); - const actual = JSON.parse(new TextDecoder().decode(data)); + const actual = JSON.parse(new TextDecoder().decode(result.artifacts.keyboard_info.data)); const expected = JSON.parse(fs.readFileSync(buildKeyboardInfoFilename, 'utf-8')); // `lastModifiedDate` is dependent on time of run (not worth mocking) diff --git a/developer/src/kmc-kmn/src/compiler/compiler.ts b/developer/src/kmc-kmn/src/compiler/compiler.ts index 317cfc757c..6878fc1b47 100644 --- a/developer/src/kmc-kmn/src/compiler/compiler.ts +++ b/developer/src/kmc-kmn/src/compiler/compiler.ts @@ -6,17 +6,12 @@ TODO: implement additional interfaces: */ // TODO: rename wasm-host? -import { UnicodeSetParser, UnicodeSet, Osk, VisualKeyboard, KvkFileReader } from '@keymanapp/common-types'; +import { UnicodeSetParser, UnicodeSet, Osk, VisualKeyboard, KvkFileReader, KeymanCompiler, KeymanCompilerArtifacts, KeymanCompilerArtifactOptional, KeymanCompilerResult, KeymanCompilerArtifact } from '@keymanapp/common-types'; import { CompilerCallbacks, CompilerEvent, CompilerOptions, KeymanFileTypes, KvkFileWriter, KvksFileReader } from '@keymanapp/common-types'; import loadWasmHost from '../import/kmcmplib/wasm-host.js'; import { CompilerMessages, mapErrorFromKmcmplib } from './kmn-compiler-messages.js'; import { WriteCompiledKeyboard } from '../kmw-compiler/kmw-compiler.js'; -export interface CompilerResultFile { - filename: string; - data: Uint8Array; -}; - // // Matches kmcmplibapi.h definitions // @@ -46,7 +41,7 @@ export const COMPILETARGETS__MASK = 0x03; /** * Data in CompilerResultExtra comes from kmcmplib */ -export interface CompilerResultExtra { +export interface KmnCompilerResultExtra { /** * A bitmask, consisting of COMPILETARGETS_KMX and/or COMPILETARGETS_JS */ @@ -61,11 +56,15 @@ export interface CompilerResultExtra { // Internal in-memory result from a successful compilation // -export interface CompilerResult { - kmx?: CompilerResultFile; - kvk?: CompilerResultFile; - js?: CompilerResultFile; - extra: CompilerResultExtra; +export interface KmnCompilerArtifacts extends KeymanCompilerArtifacts { + kmx?: KeymanCompilerArtifactOptional; + kvk?: KeymanCompilerArtifactOptional; + js?: KeymanCompilerArtifactOptional; +}; + +export interface KmnCompilerResult extends KeymanCompilerResult { + artifacts: KmnCompilerArtifacts; + extra: KmnCompilerResultExtra; displayMap?: Osk.PuaMap; }; @@ -97,18 +96,20 @@ interface MallocAndFree { let Module: any; -export class KmnCompiler implements UnicodeSetParser { - callbackID: string; // a unique numeric id added to globals with prefixed names - callbacks: CompilerCallbacks; - wasmExports: MallocAndFree; +export class KmnCompiler implements KeymanCompiler, UnicodeSetParser { + private readonly callbackID: string; // a unique numeric id added to globals with prefixed names + private callbacks: CompilerCallbacks; + private wasmExports: MallocAndFree; + private options: KmnCompilerOptions; constructor() { this.callbackID = callbackPrefix + callbackProcIdentifier.toString(); callbackProcIdentifier++; } - public async init(callbacks: CompilerCallbacks): Promise { + public async init(callbacks: CompilerCallbacks, options: KmnCompilerOptions): Promise { this.callbacks = callbacks; + this.options = {...options}; if(!Module) { try { Module = await loadWasmHost(); @@ -140,20 +141,22 @@ export class KmnCompiler implements UnicodeSetParser { return true; } - public run(infile: string, options?: KmnCompilerOptions): boolean { - let result = this.runCompiler(infile, options); - if(result) { - if(result.kmx) { - this.callbacks.fs.writeFileSync(result.kmx.filename, result.kmx.data); - } - if(result.kvk) { - this.callbacks.fs.writeFileSync(result.kvk.filename, result.kvk.data); - } - if(result.js) { - this.callbacks.fs.writeFileSync(result.js.filename, result.js.data); - } + public async write(artifacts: KmnCompilerArtifacts): Promise { + if(!artifacts) { + throw Error('artifacts must be defined'); } - return !!result; + + if(artifacts.kmx) { + this.callbacks.fs.writeFileSync(artifacts.kmx.filename, artifacts.kmx.data); + } + if(artifacts.kvk) { + this.callbacks.fs.writeFileSync(artifacts.kvk.filename, artifacts.kvk.data); + } + if(artifacts.js) { + this.callbacks.fs.writeFileSync(artifacts.js.filename, artifacts.js.data); + } + + return true; } private compilerMessageCallback = (line: number, code: number, msg: string): number => { @@ -196,9 +199,10 @@ export class KmnCompiler implements UnicodeSetParser { return 1; } - private copyWasmResult(wasm_result: any): CompilerResult { - let result: CompilerResult = { + private copyWasmResult(wasm_result: any): KmnCompilerResult { + let result: KmnCompilerResult = { // We cannot Object.assign or {...} on a wasm-defined object, so... + artifacts: {}, extra: { targets: wasm_result.extra.targets, displayMapFilename: wasm_result.extra.displayMapFilename, @@ -234,15 +238,15 @@ export class KmnCompiler implements UnicodeSetParser { return new Uint8Array(new Uint8Array(Module.HEAP8.buffer, offset, size)); } - public runCompiler(infile: string, options: KmnCompilerOptions): CompilerResult { + public async run(infile: string, outfile: string): Promise { if(!this.verifyInitialized()) { /* c8 ignore next 2 */ return null; } - options = {...baseOptions, ...options}; + const options = {...baseOptions, ...this.options}; - options.outFile = options.outFile ?? infile.replace(/\.kmn$/i, '.kmx'); + outfile = outfile ?? infile.replace(/\.kmn$/i, '.kmx'); (globalThis as any)[this.callbackID] = { message: this.compilerMessageCallback, @@ -264,11 +268,11 @@ export class KmnCompiler implements UnicodeSetParser { return null; } - const result: CompilerResult = this.copyWasmResult(wasm_result); + const result: KmnCompilerResult = this.copyWasmResult(wasm_result); if(result.extra.targets & COMPILETARGETS_KMX) { - result.kmx = { - filename: options.outFile, + result.artifacts.kmx = { + filename: outfile, data: this.copyWasmBuffer(wasm_result.kmx, wasm_result.kmxSize) }; } @@ -286,8 +290,8 @@ export class KmnCompiler implements UnicodeSetParser { } if(result.extra.kvksFilename) { - result.kvk = this.runKvkCompiler(result.extra.kvksFilename, infile, options.outFile, result.displayMap); - if(!result.kvk) { + result.artifacts.kvk = this.runKvkCompiler(result.extra.kvksFilename, infile, outfile, result.displayMap); + if(!result.artifacts.kvk) { return null; } } @@ -308,12 +312,12 @@ export class KmnCompiler implements UnicodeSetParser { if(!wasm_result.result) { return null; } - const kmw_result: CompilerResult = this.copyWasmResult(wasm_result); + const kmw_result: KmnCompilerResult = this.copyWasmResult(wasm_result); kmw_result.displayMap = result.displayMap; // we can safely re-use the kmx compile displayMap const web_kmx = this.copyWasmBuffer(wasm_result.kmx, wasm_result.kmxSize); - result.js = this.runWebCompiler(infile, options.outFile, web_kmx, result.kvk?.data, kmw_result, options); - if(!result.js) { + result.artifacts.js = this.runWebCompiler(infile, outfile, web_kmx, result.artifacts.kvk?.data, kmw_result, options); + if(!result.artifacts.js) { return null; } } @@ -338,9 +342,9 @@ export class KmnCompiler implements UnicodeSetParser { kmxFilename: string, web_kmx: Uint8Array, kvk: Uint8Array, - kmxResult: CompilerResult, + kmxResult: KmnCompilerResult, options: CompilerOptions - ): CompilerResultFile { + ): KeymanCompilerArtifact { const data = WriteCompiledKeyboard(this.callbacks, kmnFilename, web_kmx, kvk, kmxResult, options.saveDebug); if(!data) { return null; @@ -348,7 +352,7 @@ export class KmnCompiler implements UnicodeSetParser { return { filename: this.callbacks.path.join(this.callbacks.path.dirname(kmxFilename), - this.keyboardIdFromKmnFilename(kmnFilename) + KeymanFileTypes.Binary.WebKeyboard), + this.keyboardIdFromKmnFilename(kmnFilename) + KeymanFileTypes.Binary.WebKeyboard), data: new TextEncoder().encode(data) }; } diff --git a/developer/src/kmc-kmn/src/kmw-compiler/compiler-globals.ts b/developer/src/kmc-kmn/src/kmw-compiler/compiler-globals.ts index 92cba914c5..3b09a18f47 100644 --- a/developer/src/kmc-kmn/src/kmw-compiler/compiler-globals.ts +++ b/developer/src/kmc-kmn/src/kmw-compiler/compiler-globals.ts @@ -1,10 +1,10 @@ import { KMX, CompilerCallbacks, CompilerOptions } from "@keymanapp/common-types"; -import { CompilerResult } from "../compiler/compiler.js"; +import { KmnCompilerResult } from "../compiler/compiler.js"; export let FTabStop: string; export let nl: string; export let FCompilerWarningsAsErrors = false; -export let kmxResult: CompilerResult; +export let kmxResult: KmnCompilerResult; export let fk: KMX.KEYBOARD; export let FMnemonic: boolean; export let options: CompilerOptions; @@ -19,7 +19,7 @@ export function setupGlobals( _options: CompilerOptions, _tab: string, _nl: string, - _kmxResult: CompilerResult, + _kmxResult: KmnCompilerResult, _keyboard: KMX.KEYBOARD, _kmnfile: string ) { diff --git a/developer/src/kmc-kmn/src/kmw-compiler/kmw-compiler.ts b/developer/src/kmc-kmn/src/kmw-compiler/kmw-compiler.ts index dd891cd7cb..974e4fd485 100644 --- a/developer/src/kmc-kmn/src/kmw-compiler/kmw-compiler.ts +++ b/developer/src/kmc-kmn/src/kmw-compiler/kmw-compiler.ts @@ -5,7 +5,7 @@ import { JavaScript_ContextMatch, JavaScript_KeyAsString, JavaScript_Name, JavaS import { KmwCompilerMessages } from "./kmw-compiler-messages.js"; import { ValidateLayoutFile } from "./validate-layout-file.js"; import { VisualKeyboardFromFile } from "./visual-keyboard-compiler.js"; -import { CompilerResult, STORETYPE_DEBUG, STORETYPE_OPTION, STORETYPE_RESERVED } from "../compiler/compiler.js"; +import { KmnCompilerResult, STORETYPE_DEBUG, STORETYPE_OPTION, STORETYPE_RESERVED } from "../compiler/compiler.js"; function requote(s: string): string { return "'" + s.replaceAll(/(['\\])/g, "\\$1") + "'"; @@ -41,7 +41,7 @@ export function WriteCompiledKeyboard( kmnfile: string, keyboardData: Uint8Array, kvkData: Uint8Array, - kmxResult: CompilerResult, + kmxResult: KmnCompilerResult, FDebug: boolean = false ): string { let opts: CompilerOptions = { diff --git a/developer/src/kmc-kmn/test/kmw/test-kmw-compiler.ts b/developer/src/kmc-kmn/test/kmw/test-kmw-compiler.ts index a7eecda019..6732351576 100644 --- a/developer/src/kmc-kmn/test/kmw/test-kmw-compiler.ts +++ b/developer/src/kmc-kmn/test/kmw/test-kmw-compiler.ts @@ -5,7 +5,7 @@ import { dirname } from 'path'; import { fileURLToPath } from 'url'; import fs from 'fs'; import { TestCompilerCallbacks } from '@keymanapp/developer-test-helpers'; -import { CompilerResult, KmnCompiler } from '../../src/compiler/compiler.js'; +import { KmnCompilerResult, KmnCompiler } from '../../src/compiler/compiler.js'; import { ETLResult, extractTouchLayout as parseWebTestResult } from './util.js'; import { KeymanFileTypes } from '@keymanapp/common-types'; @@ -27,7 +27,10 @@ describe('KeymanWeb Compiler', function() { const kmnCompiler: KmnCompiler = new KmnCompiler(); this.beforeAll(async function() { - assert.isTrue(await kmnCompiler.init(callbacks)); + assert.isTrue(await kmnCompiler.init(callbacks, { + shouldAddCompilerVersion: false, + saveDebug: true, + })); }); this.afterEach(function() { @@ -79,18 +82,16 @@ describe('KeymanWeb Compiler', function() { }); -function run_test_keyboard(kmnCompiler: KmnCompiler, id: string): { result: CompilerResult, actualCode: string, actual: ETLResult, expectedCode: string, expected: ETLResult } { +async function run_test_keyboard(kmnCompiler: KmnCompiler, id: string): + Promise<{ result: KmnCompilerResult, actualCode: string, actual: ETLResult, expectedCode: string, expected: ETLResult }> { const filenames = generateTestFilenames(id); - let result = kmnCompiler.runCompiler(filenames.source, { - shouldAddCompilerVersion: false, - saveDebug: true, - }); + let result = await kmnCompiler.run(filenames.source, null); assert.isNotNull(result); let value = { result, - actualCode: new TextDecoder().decode(result.js.data), + actualCode: new TextDecoder().decode(result.artifacts.js.data), expectedCode: fs.readFileSync(filenames.fixture, 'utf8'), expected: null, actual: null, diff --git a/developer/src/kmc-kmn/test/test-compiler.ts b/developer/src/kmc-kmn/test/test-compiler.ts index 0b776266ce..d103625e9b 100644 --- a/developer/src/kmc-kmn/test/test-compiler.ts +++ b/developer/src/kmc-kmn/test/test-compiler.ts @@ -15,7 +15,7 @@ describe('Compiler class', function() { const compiler = new KmnCompiler(); const callbacks : any = null; // ERROR try { - await compiler.init(callbacks) + await compiler.init(callbacks, null) assert.fail('Expected exception'); } catch(e) { assert.ok(e); @@ -26,21 +26,27 @@ describe('Compiler class', function() { it('should start', async function() { const compiler = new KmnCompiler(); const callbacks = new TestCompilerCallbacks(); - assert(await compiler.init(callbacks)); + assert(await compiler.init(callbacks, null)); assert(compiler.verifyInitialized()); }); it('should compile a basic keyboard', async function() { const compiler = new KmnCompiler(); const callbacks = new TestCompilerCallbacks(); - assert(await compiler.init(callbacks)); + assert(await compiler.init(callbacks, {saveDebug: true, shouldAddCompilerVersion: false})); assert(compiler.verifyInitialized()); const fixtureName = baselineDir + 'k_000___null_keyboard.kmx'; const infile = baselineDir + 'k_000___null_keyboard.kmn'; const outFile = __dirname + '/k_000___null_keyboard.kmx'; - assert(compiler.run(infile, {saveDebug: true, outFile, shouldAddCompilerVersion: false})); + if(fs.existsSync(outFile)) { + fs.rmSync(outFile); + } + + const result = await compiler.run(infile, outFile); + assert.isNotNull(result); + assert.isTrue(await compiler.write(result.artifacts)); assert(fs.existsSync(outFile)); const outfileData = fs.readFileSync(outFile); @@ -54,7 +60,7 @@ describe('Compiler class', function() { it('should build all baseline fixtures', async function() { const compiler = new KmnCompiler(); const callbacks = new TestCompilerCallbacks(); - assert(await compiler.init(callbacks)); + assert(await compiler.init(callbacks, {saveDebug: true, shouldAddCompilerVersion: false})); assert(compiler.verifyInitialized()); const files = fs.readdirSync(baselineDir); @@ -64,7 +70,13 @@ describe('Compiler class', function() { const infile = baselineDir + file.replace(/x$/, 'n'); const outFile = __dirname + '/' + file; - assert(compiler.run(infile, {saveDebug: true, outFile, shouldAddCompilerVersion: false})); + if(fs.existsSync(outFile)) { + fs.rmSync(outFile); + } + + const result = await compiler.run(infile, outFile); + assert.isNotNull(result); + assert.isTrue(await compiler.write(result.artifacts)); assert(fs.existsSync(outFile)); const outfileData = fs.readFileSync(outFile); @@ -78,7 +90,10 @@ describe('Compiler class', function() { it('should compile a keyboard with visual keyboard', async function() { const compiler = new KmnCompiler(); const callbacks = new TestCompilerCallbacks(); - assert.isTrue(await compiler.init(callbacks)); + assert.isTrue(await compiler.init(callbacks, { + saveDebug: true, + shouldAddCompilerVersion: false, + })); assert.isTrue(compiler.verifyInitialized()); const fixtureDir = keyboardsDir + 'caps_lock_layer_3620/' @@ -89,11 +104,17 @@ describe('Compiler class', function() { const resultingKmxfile = __dirname + '/caps_lock_layer_3620.kmx'; const resultingKvkfile = __dirname + '/caps_lock_layer_3620.kvk'; - assert.isTrue(compiler.run(infile, { - saveDebug: true, - shouldAddCompilerVersion: false, - outFile: resultingKmxfile, - })); + if(fs.existsSync(resultingKmxfile)) { + fs.rmSync(resultingKmxfile); + } + + if(fs.existsSync(resultingKvkfile)) { + fs.rmSync(resultingKvkfile); + } + + const result = await compiler.run(infile, resultingKmxfile); + assert.isNotNull(result); + assert.isTrue(await compiler.write(result.artifacts)); assert.isTrue(fs.existsSync(resultingKmxfile)); assert.isTrue(fs.existsSync(resultingKvkfile)); diff --git a/developer/src/kmc-kmn/test/test-features.ts b/developer/src/kmc-kmn/test/test-features.ts index 81c9aa24ea..a030ba984c 100644 --- a/developer/src/kmc-kmn/test/test-features.ts +++ b/developer/src/kmc-kmn/test/test-features.ts @@ -5,14 +5,14 @@ import { TestCompilerCallbacks } from '@keymanapp/developer-test-helpers'; import { makePathToFixture } from './helpers/index.js'; import { KMX, KmxFileReader } from '@keymanapp/common-types'; -describe('Keyboard compiler features', async function() { +describe('Keyboard compiler features', function() { let compiler: KmnCompiler = null; let callbacks: TestCompilerCallbacks = null; this.beforeAll(async function() { compiler = new KmnCompiler(); callbacks = new TestCompilerCallbacks(); - assert(await compiler.init(callbacks)); + assert(await compiler.init(callbacks, {saveDebug: true})); assert(compiler.verifyInitialized()); }); @@ -29,15 +29,15 @@ describe('Keyboard compiler features', async function() { ]; for(const v of versions) { - it(`should build a version ${v[0]} keyboard`, function() { + it(`should build a version ${v[0]} keyboard`, async function() { const fixtureName = makePathToFixture('features', `version_${v[1]}.kmn`); - const result = compiler.runCompiler(fixtureName, {outFile: `version_${v[1]}.kmx`, saveDebug: true}); + const result = await compiler.run(fixtureName, `version_${v[1]}.kmx`); if(result === null) callbacks.printMessages(); assert.isNotNull(result); const reader = new KmxFileReader(); - const keyboard = reader.read(result.kmx.data); + const keyboard = reader.read(result.artifacts.kmx.data); assert.equal(keyboard.fileVersion, v[2]); }); } diff --git a/developer/src/kmc-kmn/test/test-messages.ts b/developer/src/kmc-kmn/test/test-messages.ts index 51e6894daa..92f8936f32 100644 --- a/developer/src/kmc-kmn/test/test-messages.ts +++ b/developer/src/kmc-kmn/test/test-messages.ts @@ -23,13 +23,13 @@ describe('CompilerMessages', function () { callbacks.clear(); const compiler = new KmnCompiler(); - assert(await compiler.init(callbacks)); + assert(await compiler.init(callbacks, {saveDebug: true, shouldAddCompilerVersion: false})); assert(compiler.verifyInitialized()); const kmnPath = makePathToFixture(...fixture); // Note: throwing away compile results (just to memory) - compiler.runCompiler(kmnPath, {saveDebug: true, shouldAddCompilerVersion: false}); + await compiler.run(kmnPath, null); if(messageId) { assert.isTrue(callbacks.hasMessage(messageId), `messageId ${messageId.toString(16)} not generated, instead got: `+JSON.stringify(callbacks.messages,null,2)); diff --git a/developer/src/kmc-kmn/test/test-wasm-uset.ts b/developer/src/kmc-kmn/test/test-wasm-uset.ts index e2bf5fcf1d..13288e10ca 100644 --- a/developer/src/kmc-kmn/test/test-wasm-uset.ts +++ b/developer/src/kmc-kmn/test/test-wasm-uset.ts @@ -25,14 +25,14 @@ describe('Compiler UnicodeSet function', function() { it('should start', async function() { const compiler = new KmnCompiler(); const callbacks = new TestCompilerCallbacks(); - assert(await compiler.init(callbacks)); + assert(await compiler.init(callbacks, null)); assert(compiler.verifyInitialized()); }); it('should compile a basic uset', async function() { const compiler = new KmnCompiler(); const callbacks = new TestCompilerCallbacks(); - assert(await compiler.init(callbacks)); + assert(await compiler.init(callbacks, null)); assert(compiler.verifyInitialized()); const pat = "[abc]"; @@ -50,7 +50,7 @@ describe('Compiler UnicodeSet function', function() { it('should compile a more complex uset', async function() { const compiler = new KmnCompiler(); const callbacks = new TestCompilerCallbacks(); - assert(await compiler.init(callbacks)); + assert(await compiler.init(callbacks, null)); assert(compiler.verifyInitialized()); const pat = "[[🙀A-C]-[CB]]"; @@ -70,7 +70,7 @@ describe('Compiler UnicodeSet function', function() { it('should compile an even more complex uset', async function() { const compiler = new KmnCompiler(); const callbacks = new TestCompilerCallbacks(); - assert(await compiler.init(callbacks)); + assert(await compiler.init(callbacks, null)); assert(compiler.verifyInitialized()); const pat = "[\\u{10FFFD}\\u{2019}\\u{22}\\u{a}\\u{ead}\\u{1F640}]"; @@ -97,7 +97,7 @@ describe('Compiler UnicodeSet function', function() { it('should fail in various ways', async function() { const compiler = new KmnCompiler(); const callbacks = new TestCompilerCallbacks(); - assert(await compiler.init(callbacks)); + assert(await compiler.init(callbacks, null)); assert(compiler.verifyInitialized()); // map from string to failing error const failures = { diff --git a/developer/src/kmc-ldml/src/compiler/compiler.ts b/developer/src/kmc-ldml/src/compiler/compiler.ts index b6f51f553b..9cd19bf19e 100644 --- a/developer/src/kmc-ldml/src/compiler/compiler.ts +++ b/developer/src/kmc-ldml/src/compiler/compiler.ts @@ -1,4 +1,4 @@ -import { LDMLKeyboardXMLSourceFileReader, LDMLKeyboard, KMXPlus, CompilerCallbacks, LDMLKeyboardTestDataXMLSourceFile, UnicodeSetParser } from '@keymanapp/common-types'; +import { LDMLKeyboardXMLSourceFileReader, LDMLKeyboard, KMXPlus, CompilerCallbacks, LDMLKeyboardTestDataXMLSourceFile, UnicodeSetParser, KeymanCompiler, KeymanCompilerResult, KeymanCompilerArtifacts, defaultCompilerOptions, KMXBuilder, KvkFileWriter, KeymanCompilerArtifactOptional } from '@keymanapp/common-types'; import { LdmlCompilerOptions } from './ldml-compiler-options.js'; import { CompilerMessages } from './messages.js'; import { BkspCompiler, TranCompiler } from './tran.js'; @@ -16,6 +16,9 @@ import KMXPlusFile = KMXPlus.KMXPlusFile; import DependencySections = KMXPlus.DependencySections; import { SectionIdent, constants } from '@keymanapp/ldml-keyboard-constants'; import { KmnCompiler } from '@keymanapp/kmc-kmn'; +import { KMXPlusMetadataCompiler } from './metadata-compiler.js'; +import { LdmlKeyboardVisualKeyboardCompiler } from './visual-keyboard-compiler.js'; +import { LdmlKeyboardKeymanWebCompiler } from './keymanweb-compiler.js'; export const SECTION_COMPILERS = [ // These are in dependency order. @@ -37,18 +40,93 @@ export const SECTION_COMPILERS = [ TranCompiler, ]; -export class LdmlKeyboardCompiler { - private readonly callbacks: CompilerCallbacks; - private readonly options: LdmlCompilerOptions; +export interface LdmlKeyboardCompilerArtifacts extends KeymanCompilerArtifacts { + kmx?: KeymanCompilerArtifactOptional; + kvk?: KeymanCompilerArtifactOptional; + js?: KeymanCompilerArtifactOptional; +}; + +export interface LdmlKeyboardCompilerResult extends KeymanCompilerResult { + artifacts: LdmlKeyboardCompilerArtifacts; +}; + +export class LdmlKeyboardCompiler implements KeymanCompiler { + private callbacks: CompilerCallbacks; + private options: LdmlCompilerOptions; // uset parser private usetparser?: UnicodeSetParser = undefined; - constructor (callbacks: CompilerCallbacks, options: LdmlCompilerOptions) { - this.options = { - ...options - }; + async init(callbacks: CompilerCallbacks, options: LdmlCompilerOptions): Promise { + this.options = {...options}; this.callbacks = callbacks; + return true; + } + + async run(inputFilename: string, outputFilename?: string): Promise { + + let compilerOptions: LdmlCompilerOptions = { + ...defaultCompilerOptions, + ...this.options, + }; + + let source = this.load(inputFilename); + if (!source) { + return null; + } + let kmx = await this.compile(source); + if (!kmx) { + return null; + } + + // In order for the KMX file to be loaded by non-KMXPlus components, it is helpful + // to duplicate some of the metadata + KMXPlusMetadataCompiler.addKmxMetadata(kmx.kmxplus, kmx.keyboard, compilerOptions); + + // Use the builder to generate the binary output file + const builder = new KMXBuilder(kmx, compilerOptions.saveDebug); + const kmx_binary = builder.compile(); + + const vkcompiler = new LdmlKeyboardVisualKeyboardCompiler(this.callbacks); + const vk = vkcompiler.compile(source); + const writer = new KvkFileWriter(); + const kvk_binary = writer.write(vk); + + // Note: we could have a step of generating source files here + // KvksFileWriter()... + // const tlcompiler = new kmc.TouchLayoutCompiler(); + // const tl = tlcompiler.compile(source); + // const tlwriter = new TouchLayoutFileWriter(); + const kmwcompiler = new LdmlKeyboardKeymanWebCompiler(this.callbacks, compilerOptions); + const kmw_string = kmwcompiler.compile(inputFilename, source); + const encoder = new TextEncoder(); + const kmw_binary = encoder.encode(kmw_string); + + outputFilename = outputFilename ?? inputFilename.replace(/\.xml$/, '.kmx'); + + return { + artifacts: { + kmx: { data: kmx_binary, filename: outputFilename }, + kvk: { data: kvk_binary, filename: outputFilename.replace(/\.kmx$/, '.kvk') }, + js: { data: kmw_binary, filename: outputFilename.replace(/\.kmx$/, '.js') }, + } + }; + } + + async write(artifacts: LdmlKeyboardCompilerArtifacts): Promise { + if(artifacts.kmx) { + this.callbacks.fs.writeFileSync(artifacts.kmx.filename, artifacts.kmx.data); + } + + if(artifacts.kvk) { + this.callbacks.fs.writeFileSync(artifacts.kvk.filename, artifacts.kvk.data); + } + + if(artifacts.js) { + this.callbacks.fs.writeFileSync(artifacts.js.filename, artifacts.js.data); + } + + return true; } /** @@ -59,7 +137,7 @@ export class LdmlKeyboardCompiler { if (this.usetparser === undefined) { // initialize const compiler = new KmnCompiler(); - const ok = await compiler.init(this.callbacks); + const ok = await compiler.init(this.callbacks, null); if (ok) { this.usetparser = compiler; } else { diff --git a/developer/src/kmc-ldml/test/helpers/index.ts b/developer/src/kmc-ldml/test/helpers/index.ts index 8f6d73ef43..53406873a4 100644 --- a/developer/src/kmc-ldml/test/helpers/index.ts +++ b/developer/src/kmc-ldml/test/helpers/index.ts @@ -112,14 +112,16 @@ async function loadDepsFor(sections: DependencySections, parentCompiler: Section } } -export function loadTestdata(inputFilename: string, options: LdmlCompilerOptions) : LDMLKeyboardTestDataXMLSourceFile { - const k = new LdmlKeyboardCompiler(compilerTestCallbacks, options); +export async function loadTestData(inputFilename: string, options: LdmlCompilerOptions) : Promise { + const k = new LdmlKeyboardCompiler(); + assert.isTrue(await k.init(compilerTestCallbacks, options)); const source = k.loadTestData(inputFilename); return source; } export async function compileKeyboard(inputFilename: string, options: LdmlCompilerOptions, validateMessages?: CompilerEvent[], expectFailValidate?: boolean, compileMessages?: CompilerEvent[]): Promise { - const k = new LdmlKeyboardCompiler(compilerTestCallbacks, options); + const k = new LdmlKeyboardCompiler(); + assert.isTrue(await k.init(compilerTestCallbacks, options)); const source = k.load(inputFilename); checkMessages(); assert.isNotNull(source, 'k.load should not have returned null'); @@ -151,7 +153,8 @@ export async function compileKeyboard(inputFilename: string, options: LdmlCompil } export async function compileVisualKeyboard(inputFilename: string, options: LdmlCompilerOptions): Promise { - const k = new LdmlKeyboardCompiler(compilerTestCallbacks, options); + const k = new LdmlKeyboardCompiler(); + assert.isTrue(await k.init(compilerTestCallbacks, options)); const source = k.load(inputFilename); checkMessages(); assert.isNotNull(source, 'k.load should not have returned null'); @@ -254,7 +257,7 @@ async function getTestUnicodeSetParser(callbacks: CompilerCallbacks): Promise(code, expected); }); - it('should handle non existent files', () => { + it('should handle non existent files', async () => { const filename = 'DOES_NOT_EXIST.xml'; - const k = new LdmlKeyboardCompiler(compilerTestCallbacks, { ...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false }); + const k = new LdmlKeyboardCompiler(); + await k.init(compilerTestCallbacks, { ...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false }); const source = k.load(filename); assert.notOk(source, `Trying to load(${filename})`); }); - it('should handle unparseable files', () => { + it('should handle unparseable files', async () => { const filename = makePathToFixture('basic-kvk.txt'); // not an .xml file - const k = new LdmlKeyboardCompiler(compilerTestCallbacks, { ...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false }); + const k = new LdmlKeyboardCompiler(); + await k.init(compilerTestCallbacks, { ...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false }); const source = k.load(filename); assert.notOk(source, `Trying to load(${filename})`); }); - it('should handle not-valid files', () => { + it('should handle not-valid files', async () => { const filename = makePathToFixture('test-fr.xml'); // not a keyboard .xml file - const k = new LdmlKeyboardCompiler(compilerTestCallbacks, { ...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false }); + const k = new LdmlKeyboardCompiler(); + await k.init(compilerTestCallbacks, { ...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false }); const source = k.load(filename); assert.notOk(source, `Trying to load(${filename})`); }); - it('should handle non existent test files', () => { + it('should handle non existent test files', async () => { const filename = 'DOES_NOT_EXIST.xml'; - const k = new LdmlKeyboardCompiler(compilerTestCallbacks, { ...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false }); + const k = new LdmlKeyboardCompiler(); + await k.init(compilerTestCallbacks, { ...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false }); const source = k.loadTestData(filename); assert.notOk(source, `Trying to loadTestData(${filename})`); }); - it('should handle unparseable test files', () => { + it('should handle unparseable test files', async () => { const filename = makePathToFixture('basic-kvk.txt'); // not an .xml file - const k = new LdmlKeyboardCompiler(compilerTestCallbacks, { ...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false }); + const k = new LdmlKeyboardCompiler(); + await k.init(compilerTestCallbacks, { ...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false }); const source = k.load(filename); assert.notOk(source, `Trying to loadTestData(${filename})`); }); diff --git a/developer/src/kmc-ldml/test/test-keymanweb-compiler.ts b/developer/src/kmc-ldml/test/test-keymanweb-compiler.ts index e1a570ea12..9a14b38c33 100644 --- a/developer/src/kmc-ldml/test/test-keymanweb-compiler.ts +++ b/developer/src/kmc-ldml/test/test-keymanweb-compiler.ts @@ -16,7 +16,8 @@ describe('LdmlKeyboardKeymanWebCompiler', function() { // Load input data; we'll use the LDML keyboard compiler loader to save us // effort here - const k = new LdmlKeyboardCompiler(compilerTestCallbacks, {...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false}); + const k = new LdmlKeyboardCompiler(); + await k.init(compilerTestCallbacks, {...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false}); const source = k.load(inputFilename); checkMessages(); assert.isNotNull(source, 'k.load should not have returned null'); diff --git a/developer/src/kmc-ldml/test/test-testdata-e2e.ts b/developer/src/kmc-ldml/test/test-testdata-e2e.ts index eea3631759..cf23352328 100644 --- a/developer/src/kmc-ldml/test/test-testdata-e2e.ts +++ b/developer/src/kmc-ldml/test/test-testdata-e2e.ts @@ -1,7 +1,7 @@ import { readFileSync } from 'fs'; import 'mocha'; import {assert} from 'chai'; -import {compilerTestOptions, loadTestdata, makePathToFixture} from './helpers/index.js'; +import {compilerTestOptions, loadTestData, makePathToFixture} from './helpers/index.js'; describe('testdata-tests', function() { this.slow(500); // 0.5 sec -- json schema validation takes a while @@ -15,7 +15,7 @@ describe('testdata-tests', function() { const jsonFilename = makePathToFixture('test-fr.json'); // Compile the keyboard - const testData = loadTestdata(inputFilename, {...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false}); + const testData = await loadTestData(inputFilename, {...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false}); assert.isNotNull(testData); const jsonData = JSON.parse(readFileSync(jsonFilename, 'utf-8')); diff --git a/developer/src/kmc-model-info/src/model-info-compiler.ts b/developer/src/kmc-model-info/src/model-info-compiler.ts index 8a7101e86f..934ffe3200 100644 --- a/developer/src/kmc-model-info/src/model-info-compiler.ts +++ b/developer/src/kmc-model-info/src/model-info-compiler.ts @@ -5,7 +5,7 @@ import { minKeymanVersion } from "./min-keyman-version.js"; import { ModelInfoFile } from "./model-info-file.js"; -import { CompilerCallbacks, KmpJsonFile } from "@keymanapp/common-types"; +import { CompilerCallbacks, CompilerOptions, KeymanCompiler, KeymanCompilerArtifact, KeymanCompilerArtifacts, KeymanCompilerResult, KmpJsonFile } from "@keymanapp/common-types"; import { ModelInfoCompilerMessages } from "./messages.js"; import { validateMITLicense } from "@keymanapp/developer-utils"; @@ -39,8 +39,30 @@ export class ModelInfoSources { }; /* c8 ignore stop */ -export class ModelInfoCompiler { - constructor(private callbacks: CompilerCallbacks) { +export interface ModelInfoCompilerOptions extends CompilerOptions { + sources: ModelInfoSources; +}; + +export interface ModelInfoCompilerArtifacts extends KeymanCompilerArtifacts { + model_info: KeymanCompilerArtifact; +}; + +export interface ModelInfoCompilerResult extends KeymanCompilerResult { + artifacts: ModelInfoCompilerArtifacts; +}; + + +export class ModelInfoCompiler implements KeymanCompiler { + private callbacks: CompilerCallbacks; + private options: ModelInfoCompilerOptions; + + constructor() { + } + + public async init(callbacks: CompilerCallbacks, options: ModelInfoCompilerOptions): Promise { + this.callbacks = callbacks; + this.options = {...options}; + return true; } /** @@ -51,9 +73,8 @@ export class ModelInfoCompiler { * * @param sources Details on files from which to extract additional metadata */ - writeModelMetadataFile( - sources: ModelInfoSources - ): Uint8Array { + public async run(inputFilename: string, outputFilename?: string): Promise { + const sources = this.options.sources; /* * Model info looks like this: @@ -171,7 +192,22 @@ export class ModelInfoCompiler { } const jsonOutput = JSON.stringify(model_info, null, 2); - return new TextEncoder().encode(jsonOutput); + const data = new TextEncoder().encode(jsonOutput); + const result: ModelInfoCompilerResult = { + artifacts: { + model_info: { + data, + filename: outputFilename ?? inputFilename.replace(/\.kpj$/, '.model_info') + } + } + }; + + return result; + } + + public async write(artifacts: ModelInfoCompilerArtifacts): Promise { + this.callbacks.fs.writeFileSync(artifacts.model_info.filename, artifacts.model_info.data); + return true; } private isLicenseMIT(filename: string) { diff --git a/developer/src/kmc-model-info/test/test-model-info-compiler.ts b/developer/src/kmc-model-info/test/test-model-info-compiler.ts index d74a2f5c60..729eca3ac9 100644 --- a/developer/src/kmc-model-info/test/test-model-info-compiler.ts +++ b/developer/src/kmc-model-info/test/test-model-info-compiler.ts @@ -13,16 +13,18 @@ beforeEach(function() { }); describe('model-info-compiler', function () { - it('compile a .model_info file correctly', function() { + it('compile a .model_info file correctly', async function() { + const kpjFilename = makePathToFixture('sil.cmo.bw', 'sil.cmo.bw.model.kpj'); const kpsFilename = makePathToFixture('sil.cmo.bw', 'source', 'sil.cmo.bw.model.kps'); const kmpFileName = makePathToFixture('sil.cmo.bw', 'build', 'sil.cmo.bw.model.kmp'); const buildModelInfoFilename = makePathToFixture('sil.cmo.bw', 'build', 'sil.cmo.bw.model_info'); - const kmpCompiler = new KmpCompiler(callbacks); + const kmpCompiler = new KmpCompiler(); + assert.isTrue(await kmpCompiler.init(callbacks, {})); const kmpJsonData = kmpCompiler.transformKpsToKmpObject(kpsFilename); const modelFileName = makePathToFixture('sil.cmo.bw', 'build', 'sil.cmo.bw.model.js'); - const data = (new ModelInfoCompiler(callbacks)).writeModelMetadataFile({ + const sources = { kmpFileName, kmpJsonData, model_id: 'sil.cmo.bw', @@ -30,13 +32,16 @@ describe('model-info-compiler', function () { sourcePath: 'release/sil/sil.cmo.bw', kpsFilename, forPublishing: true, - }); - if(data == null) { + }; + const compiler = new ModelInfoCompiler(); + assert.isTrue(await compiler.init(callbacks, {sources})); + const result = await compiler.run(kpjFilename, null); + if(result == null) { callbacks.printMessages(); } - assert.isNotNull(data); + assert.isNotNull(result); - const actual = JSON.parse(new TextDecoder().decode(data)); + const actual = JSON.parse(new TextDecoder().decode(result.artifacts.model_info.data)); let expected = JSON.parse(fs.readFileSync(buildModelInfoFilename, 'utf-8')); // `lastModifiedDate` is dependent on time of run (not worth mocking) diff --git a/developer/src/kmc-model/src/lexical-model-compiler.ts b/developer/src/kmc-model/src/lexical-model-compiler.ts index 0edd818968..8785ed32c4 100644 --- a/developer/src/kmc-model/src/lexical-model-compiler.ts +++ b/developer/src/kmc-model/src/lexical-model-compiler.ts @@ -2,20 +2,102 @@ lexical-model-compiler.ts: base file for lexical model compiler. */ -import * as ts from "typescript"; +import ts from "typescript"; import { createTrieDataStructure } from "./build-trie.js"; import { ModelDefinitions } from "./model-definitions.js"; import {decorateWithJoin} from "./join-word-breaker-decorator.js"; import {decorateWithScriptOverrides} from "./script-overrides-decorator.js"; import { LexicalModelSource, WordBreakerSpec, SimpleWordBreakerSpec } from "./lexical-model.js"; -import { ModelCompilerError, ModelCompilerMessages } from "./model-compiler-errors.js"; +import { ModelCompilerError, ModelCompilerMessageContext, ModelCompilerMessages } from "./model-compiler-errors.js"; import { callbacks, setCompilerCallbacks } from "./compiler-callbacks.js"; -import { CompilerCallbacks } from "@keymanapp/common-types"; +import { CompilerCallbacks, CompilerOptions, KeymanCompiler, KeymanCompilerArtifact, KeymanCompilerArtifacts, KeymanCompilerResult } from "@keymanapp/common-types"; -export default class LexicalModelCompiler { +/** + * An ECMAScript module as emitted by the TypeScript compiler. + */ +interface ES2015Module { + /** This is always true. */ + __esModule: boolean; + 'default'?: unknown; +}; - constructor(callbacks: CompilerCallbacks) { +export interface LexicalModelCompilerArtifacts extends KeymanCompilerArtifacts { + js: KeymanCompilerArtifact; +}; + +export interface LexicalModelCompilerResult extends KeymanCompilerResult { + artifacts: LexicalModelCompilerArtifacts; +}; + +export class LexicalModelCompiler implements KeymanCompiler { + + async init(callbacks: CompilerCallbacks, _options: CompilerOptions): Promise { setCompilerCallbacks(callbacks); + return true; + } + + async run(inputFilename: string, outputFilename?: string): Promise { + try { + let modelSource = this.loadFromFilename(inputFilename); + let containingDirectory = callbacks.path.dirname(inputFilename); + let code = this.generateLexicalModelCode('', modelSource, containingDirectory); + const result: LexicalModelCompilerResult = { + artifacts: { + js: { + data: new TextEncoder().encode(code), + filename: outputFilename ?? inputFilename.replace(/\.model\.ts$/, '.model.js') + } + } + } + return result; + } catch(e) { + callbacks.reportMessage( + e instanceof ModelCompilerError + ? e.event + : ModelCompilerMessages.Fatal_UnexpectedException({e:e}) + ); + return null; + } + } + + async write(artifacts: LexicalModelCompilerArtifacts): Promise { + callbacks.fs.writeFileSync(artifacts.js.filename, artifacts.js.data); + return true; + } + + /** + * Loads a lexical model's source module from the given filename. + * + * @param filename path to the model source file. + */ + public loadFromFilename(filename: string): LexicalModelSource { + + let sourceCode = new TextDecoder().decode(callbacks.loadFile(filename)); + // Compile the module to JavaScript code. + // NOTE: transpile module does a very simple TS to JS compilation. + // It DOES NOT check for types! + let compilationOutput = ts.transpile(sourceCode, { + // Our runtime only supports ES3 with Node/CommonJS modules on Android 5.0. + // When we drop Android 5.0 support, we can update this to a `ScriptTarget` + // matrix against target version of Keyman, here and in + // lexical-model-compiler.ts. + target: ts.ScriptTarget.ES3, + module: ts.ModuleKind.CommonJS, + }); + // Turn the module into a function in which we can inject a global. + let moduleCode = '(function(exports){' + compilationOutput + '})'; + + // Run the module; its exports will be assigned to `moduleExports`. + let moduleExports: Partial = {}; + let module = eval(moduleCode); + module(moduleExports); + + if (!moduleExports['__esModule'] || !moduleExports['default']) { + ModelCompilerMessageContext.filename = filename; + throw new ModelCompilerError(ModelCompilerMessages.Error_NoDefaultExport()); + } + + return moduleExports['default'] as LexicalModelSource; } /** diff --git a/developer/src/kmc-model/src/main.ts b/developer/src/kmc-model/src/main.ts index 667253353d..68fae091e3 100644 --- a/developer/src/kmc-model/src/main.ts +++ b/developer/src/kmc-model/src/main.ts @@ -1,79 +1 @@ -import { CompilerCallbacks } from '@keymanapp/common-types'; -import ts from 'typescript'; -import { setCompilerCallbacks } from './compiler-callbacks.js'; - -import LexicalModelCompiler from './lexical-model-compiler.js'; -import { LexicalModelSource } from './lexical-model.js'; -import { ModelCompilerError, ModelCompilerMessageContext, ModelCompilerMessages } from './model-compiler-errors.js'; - -export { default as LexicalModelCompiler } from './lexical-model-compiler.js'; - -/** - * Compiles a model.ts file, using paths relative to its location. - * - * @param filename path to model.ts source. - * @return model source code, or null on error - */ -export function compileModel(filename: string, callbacks: CompilerCallbacks): string { - setCompilerCallbacks(callbacks); - - try { - let modelSource = loadFromFilename(filename, callbacks); - let containingDirectory = callbacks.path.dirname(filename); - - return (new LexicalModelCompiler(callbacks)) - .generateLexicalModelCode('', modelSource, containingDirectory); - } catch(e) { - callbacks.reportMessage( - e instanceof ModelCompilerError - ? e.event - : ModelCompilerMessages.Fatal_UnexpectedException({e:e}) - ); - } - return null; -} - -/** - * An ECMAScript module as emitted by the TypeScript compiler. - */ -interface ES2015Module { - /** This is always true. */ - __esModule: boolean; - 'default'?: unknown; -} - -/** - * Loads a lexical model's source module from the given filename. - * - * @param filename path to the model source file. - */ -export function loadFromFilename(filename: string, callbacks: CompilerCallbacks): LexicalModelSource { - setCompilerCallbacks(callbacks); - - let sourceCode = new TextDecoder().decode(callbacks.loadFile(filename)); - // Compile the module to JavaScript code. - // NOTE: transpile module does a very simple TS to JS compilation. - // It DOES NOT check for types! - let compilationOutput = ts.transpile(sourceCode, { - // Our runtime only supports ES3 with Node/CommonJS modules on Android 5.0. - // When we drop Android 5.0 support, we can update this to a `ScriptTarget` - // matrix against target version of Keyman, here and in - // lexical-model-compiler.ts. - target: ts.ScriptTarget.ES3, - module: ts.ModuleKind.CommonJS, - }); - // Turn the module into a function in which we can inject a global. - let moduleCode = '(function(exports){' + compilationOutput + '})'; - - // Run the module; its exports will be assigned to `moduleExports`. - let moduleExports: Partial = {}; - let module = eval(moduleCode); - module(moduleExports); - - if (!moduleExports['__esModule'] || !moduleExports['default']) { - ModelCompilerMessageContext.filename = filename; - throw new ModelCompilerError(ModelCompilerMessages.Error_NoDefaultExport()); - } - - return moduleExports['default'] as LexicalModelSource; -} +export { LexicalModelCompiler } from './lexical-model-compiler.js'; diff --git a/developer/src/kmc-model/test/test-compile-model-with-pseudoclosure.ts b/developer/src/kmc-model/test/test-compile-model-with-pseudoclosure.ts index d6fcec3959..08247e17c1 100644 --- a/developer/src/kmc-model/test/test-compile-model-with-pseudoclosure.ts +++ b/developer/src/kmc-model/test/test-compile-model-with-pseudoclosure.ts @@ -1,4 +1,4 @@ -import LexicalModelCompiler from '../src/lexical-model-compiler.js'; +import { LexicalModelCompiler } from '../src/lexical-model-compiler.js'; import {assert} from 'chai'; import 'mocha'; @@ -26,8 +26,9 @@ describe('LexicalModelCompiler - pseudoclosure compilation + use', function () { } }; - it('variant 1: applyCasing prepends symbols, searchTermToKey removes them', function() { - let compiler = new LexicalModelCompiler(callbacks); + it('variant 1: applyCasing prepends symbols, searchTermToKey removes them', async function() { + let compiler = new LexicalModelCompiler(); + assert.isTrue(await compiler.init(callbacks, null)); let code = compiler.generateLexicalModelCode(MODEL_ID, { format: 'trie-1.0', sources: ['wordlist.tsv'], @@ -80,8 +81,9 @@ describe('LexicalModelCompiler - pseudoclosure compilation + use', function () { assert.isNotNull(compilation.exportedModel); }); - it('variant 2: applyCasing prepends symbols, searchTermToKey keeps them', function() { - let compiler = new LexicalModelCompiler(callbacks); + it('variant 2: applyCasing prepends symbols, searchTermToKey keeps them', async function() { + let compiler = new LexicalModelCompiler(); + assert.isTrue(await compiler.init(callbacks, null)); let code = compiler.generateLexicalModelCode(MODEL_ID, { format: 'trie-1.0', sources: ['wordlist.tsv'], @@ -124,8 +126,9 @@ describe('LexicalModelCompiler - pseudoclosure compilation + use', function () { assert.isNotNull(compilation.exportedModel); }); - it('variant 3: applyCasing prepends symbols, default searchTermToKey', function() { - let compiler = new LexicalModelCompiler(callbacks); + it('variant 3: applyCasing prepends symbols, default searchTermToKey', async function() { + let compiler = new LexicalModelCompiler(); + assert.isTrue(await compiler.init(callbacks, null)); let code = compiler.generateLexicalModelCode(MODEL_ID, { format: 'trie-1.0', sources: ['wordlist.tsv'], @@ -164,8 +167,9 @@ describe('LexicalModelCompiler - pseudoclosure compilation + use', function () { }); describe('relying on default applyCasing + searchTermToKey', function() { - it('languageUsesCasing: true', function() { - let compiler = new LexicalModelCompiler(callbacks); + it('languageUsesCasing: true', async function() { + let compiler = new LexicalModelCompiler(); + assert.isTrue(await compiler.init(callbacks, null)); let code = compiler.generateLexicalModelCode(MODEL_ID, { format: 'trie-1.0', sources: ['wordlist.tsv'], @@ -188,8 +192,9 @@ describe('LexicalModelCompiler - pseudoclosure compilation + use', function () { assert.isNotNull(compilation.exportedModel); }); - it('languageUsesCasing: false', function() { - let compiler = new LexicalModelCompiler(callbacks); + it('languageUsesCasing: false', async function() { + let compiler = new LexicalModelCompiler(); + assert.isTrue(await compiler.init(callbacks, null)); let code = compiler.generateLexicalModelCode(MODEL_ID, { format: 'trie-1.0', sources: ['wordlist.tsv'], @@ -213,8 +218,9 @@ describe('LexicalModelCompiler - pseudoclosure compilation + use', function () { assert.isNotNull(compilation.exportedModel); }); - it('languageUsesCasing: undefined', function() { - let compiler = new LexicalModelCompiler(callbacks); + it('languageUsesCasing: undefined', async function() { + let compiler = new LexicalModelCompiler(); + assert.isTrue(await compiler.init(callbacks, null)); let code = compiler.generateLexicalModelCode(MODEL_ID, { format: 'trie-1.0', sources: ['wordlist.tsv'] diff --git a/developer/src/kmc-model/test/test-compile-model.ts b/developer/src/kmc-model/test/test-compile-model.ts index 2db28b6e09..e3074d697a 100644 --- a/developer/src/kmc-model/test/test-compile-model.ts +++ b/developer/src/kmc-model/test/test-compile-model.ts @@ -1,12 +1,12 @@ import 'mocha'; import {assert} from 'chai'; -import {compileModel} from '../src/main.js'; +import { LexicalModelCompiler } from '../src/main.js'; import {makePathToFixture, compileModelSourceCode, CompilationResult} from './helpers/index.js'; import { TestCompilerCallbacks } from '@keymanapp/developer-test-helpers'; import { KeymanFileTypes } from '@keymanapp/common-types'; -describe('compileModel', function () { +describe('LexicalModelCompiler', function () { let callbacks = new TestCompilerCallbacks(); // Try to compile ALL of the correct models. @@ -22,9 +22,14 @@ describe('compileModel', function () { for (let modelID of MODELS) { let modelPath = makePathToFixture(modelID, modelID + KeymanFileTypes.Source.Model); - it(`should compile ${modelID}`, function () { - let code = compileModel(modelPath, callbacks); + it(`should compile ${modelID}`, async function () { + const compiler = new LexicalModelCompiler(); + assert.isTrue(await compiler.init(callbacks, null)); + const result = await compiler.run(modelPath, null); callbacks.printMessages(); + assert.isNotNull(result); + const decoder = new TextDecoder(); + const code = decoder.decode(result.artifacts.js.data); let r = compileModelSourceCode(code); let compilation = r as CompilationResult; diff --git a/developer/src/kmc-model/test/test-compile-trie.ts b/developer/src/kmc-model/test/test-compile-trie.ts index 544924da3e..5d80b72ac1 100644 --- a/developer/src/kmc-model/test/test-compile-trie.ts +++ b/developer/src/kmc-model/test/test-compile-trie.ts @@ -1,4 +1,4 @@ -import LexicalModelCompiler from '../src/lexical-model-compiler.js'; +import { LexicalModelCompiler } from '../src/lexical-model-compiler.js'; import {assert} from 'chai'; import 'mocha'; @@ -14,11 +14,12 @@ describe('LexicalModelCompiler', function () { }); describe('#generateLexicalModelCode', function () { - it('should compile a trivial word list', function () { + it('should compile a trivial word list', async function () { const MODEL_ID = 'example.qaa.trivial'; const PATH = makePathToFixture(MODEL_ID); - let compiler = new LexicalModelCompiler(callbacks); + let compiler = new LexicalModelCompiler(); + assert.isTrue(await compiler.init(callbacks, null)); let code = compiler.generateLexicalModelCode(MODEL_ID, { format: 'trie-1.0', sources: ['wordlist.tsv'] @@ -37,11 +38,12 @@ describe('LexicalModelCompiler', function () { assert.match(code, /\bwordBreaker\b["']?:\s*wordBreakers\b/); }); - it('should compile a word list exported by Microsoft Excel', function () { + it('should compile a word list exported by Microsoft Excel', async function () { const MODEL_ID = 'example.qaa.utf16le'; const PATH = makePathToFixture(MODEL_ID); - let compiler = new LexicalModelCompiler(callbacks); + let compiler = new LexicalModelCompiler(); + assert.isTrue(await compiler.init(callbacks, null)); let code = compiler.generateLexicalModelCode(MODEL_ID, { format: 'trie-1.0', sources: ['wordlist.txt'] @@ -58,11 +60,12 @@ describe('LexicalModelCompiler', function () { }); }); - it('should compile a word list with a custom word breaking function', function () { + it('should compile a word list with a custom word breaking function', async function () { const MODEL_ID = 'example.qaa.trivial'; const PATH = makePathToFixture(MODEL_ID); - let compiler = new LexicalModelCompiler(callbacks); + let compiler = new LexicalModelCompiler(); + assert.isTrue(await compiler.init(callbacks, null)); let code = compiler.generateLexicalModelCode(MODEL_ID, { format: 'trie-1.0', sources: ['wordlist.tsv'], @@ -81,11 +84,12 @@ describe('LexicalModelCompiler', function () { assert.match(code, /\bwordBreaker\b["']?:\s+function\b/); }); - it('should not generate unpaired surrogate code units', function () { + it('should not generate unpaired surrogate code units', async function () { const MODEL_ID = 'example.qaa.smp'; const PATH = makePathToFixture(MODEL_ID); - let compiler = new LexicalModelCompiler(callbacks); + let compiler = new LexicalModelCompiler(); + assert.isTrue(await compiler.init(callbacks, null)); let code = compiler.generateLexicalModelCode(MODEL_ID, { format: 'trie-1.0', sources: ['wordlist.tsv'] @@ -115,10 +119,11 @@ describe('LexicalModelCompiler', function () { assert.match(code, /\btotalWeight\b["']?:\s*27596\b/); }); - it('should include the source code of its search term to key function', function () { + it('should include the source code of its search term to key function', async function () { const MODEL_ID = 'example.qaa.trivial'; const PATH = makePathToFixture(MODEL_ID); - let compiler = new LexicalModelCompiler(callbacks); + let compiler = new LexicalModelCompiler(); + assert.isTrue(await compiler.init(callbacks, null)); let code = compiler.generateLexicalModelCode(MODEL_ID, { format: 'trie-1.0', sources: ['wordlist.tsv'] diff --git a/developer/src/kmc-model/test/test-punctuation.ts b/developer/src/kmc-model/test/test-punctuation.ts index 133eed5fed..f3ad64433e 100644 --- a/developer/src/kmc-model/test/test-punctuation.ts +++ b/developer/src/kmc-model/test/test-punctuation.ts @@ -1,4 +1,4 @@ -import LexicalModelCompiler from '../src/lexical-model-compiler.js'; +import { LexicalModelCompiler } from '../src/lexical-model-compiler.js'; import {assert} from 'chai'; import 'mocha'; @@ -10,10 +10,11 @@ describe('LexicalModelCompiler', function () { const MODEL_ID = 'example.qaa.trivial'; const PATH = makePathToFixture(MODEL_ID); - it('should compile punctuation into the generated code', function () { + it('should compile punctuation into the generated code', async function () { const callbacks = new TestCompilerCallbacks(); - let compiler = new LexicalModelCompiler(callbacks); + let compiler = new LexicalModelCompiler(); + assert.isTrue(await compiler.init(callbacks, null)); let code = compiler.generateLexicalModelCode(MODEL_ID, { format: 'trie-1.0', sources: ['wordlist.tsv'], diff --git a/developer/src/kmc-package/src/compiler/kmp-compiler.ts b/developer/src/kmc-package/src/compiler/kmp-compiler.ts index 3dce88c9d5..b9993713d0 100644 --- a/developer/src/kmc-package/src/compiler/kmp-compiler.ts +++ b/developer/src/kmc-package/src/compiler/kmp-compiler.ts @@ -2,7 +2,7 @@ import * as xml2js from 'xml2js'; import JSZip from 'jszip'; import KEYMAN_VERSION from "@keymanapp/keyman-version"; -import { KmpJsonFile, KpsFile, SchemaValidators, CompilerCallbacks, KeymanFileTypes, KvkFile } from '@keymanapp/common-types'; +import { KmpJsonFile, KpsFile, SchemaValidators, CompilerCallbacks, KeymanFileTypes, KvkFile, KeymanCompiler, CompilerOptions, KeymanCompilerResult, KeymanCompilerArtifacts, KeymanCompilerArtifact } from '@keymanapp/common-types'; import { CompilerMessages } from './messages.js'; import { PackageMetadataCollector } from './package-metadata-collector.js'; import { KmpInfWriter } from './kmp-inf-writer.js'; @@ -11,6 +11,7 @@ import { MIN_LM_FILEVERSION_KMP_JSON, PackageVersionValidator } from './package- import { PackageKeyboardTargetValidator } from './package-keyboard-target-validator.js'; import { PackageMetadataUpdater } from './package-metadata-updater.js'; import { markdownToHTML } from './markdown.js'; +import { PackageValidation } from './package-validation.js'; const KMP_JSON_FILENAME = 'kmp.json'; const KMP_INF_FILENAME = 'kmp.inf'; @@ -20,9 +21,68 @@ const KMP_INF_FILENAME = 'kmp.inf'; // this filename for existing keyboard packages. const WELCOME_HTM_FILENAME = 'welcome.htm'; -export class KmpCompiler { +export interface KmpCompilerOptions extends CompilerOptions { + // Note: WindowsPackageInstallerCompilerOptions extends KmpCompilerOptions, so + // be careful when modifying this interface +}; - constructor(private callbacks: CompilerCallbacks) { +export interface KmpCompilerArtifacts extends KeymanCompilerArtifacts { + kmp: KeymanCompilerArtifact; +}; + +export interface KmpCompilerResult extends KeymanCompilerResult { + artifacts: KmpCompilerArtifacts; +}; + +export class KmpCompiler implements KeymanCompiler { + private callbacks: CompilerCallbacks; + private options: KmpCompilerOptions; + + public async init(callbacks: CompilerCallbacks, options: KmpCompilerOptions): Promise { + this.callbacks = callbacks; + this.options = options ? {...options} : {}; + return true; + } + + public async run(inputFilename: string, outputFilename?: string): Promise { + const kmpJsonData = this.transformKpsToKmpObject(inputFilename); + if(!kmpJsonData) { + return null; + } + + // + // Validate the package file + // + + const validation = new PackageValidation(this.callbacks, this.options); + if(!validation.validate(inputFilename, kmpJsonData)) { + return null; + } + + // + // Build the .kmp package file + // + + const data = await this.buildKmpFile(inputFilename, kmpJsonData); + if(!data) { + return null; + } + + const result: KmpCompilerResult = { + artifacts: { + kmp: { + data, + filename: outputFilename ?? inputFilename.replace(/\.kps$/, '.kmp') + } + } + } + + return result; + } + + public async write(artifacts: KmpCompilerArtifacts): Promise { + this.callbacks.fs.writeFileSync(artifacts.kmp.filename, artifacts.kmp.data); + return true; } public transformKpsToKmpObject(kpsFilename: string): KmpJsonFile.KmpJsonFile { @@ -346,7 +406,7 @@ export class KmpCompiler { * @param kpsFilename - Filename of the kps, not read, used only for calculating relative paths * @param kmpJsonData - The kmp.json Object */ - public buildKmpFile(kpsFilename: string, kmpJsonData: KmpJsonFile.KmpJsonFile): Promise { + public buildKmpFile(kpsFilename: string, kmpJsonData: KmpJsonFile.KmpJsonFile): Promise { const zip = JSZip(); @@ -438,7 +498,7 @@ export class KmpCompiler { } // Generate kmp file - return zip.generateAsync({type: 'binarystring', compression:'DEFLATE'}); + return zip.generateAsync({type:'uint8array', compression:'DEFLATE'}); } private buildKmpInf(data: KmpJsonFile.KmpJsonFile): Uint8Array { diff --git a/developer/src/kmc-package/src/compiler/windows-package-installer-compiler.ts b/developer/src/kmc-package/src/compiler/windows-package-installer-compiler.ts index 1a5f4ebe6b..8ac54572c1 100644 --- a/developer/src/kmc-package/src/compiler/windows-package-installer-compiler.ts +++ b/developer/src/kmc-package/src/compiler/windows-package-installer-compiler.ts @@ -11,9 +11,9 @@ */ import JSZip from 'jszip'; -import { CompilerCallbacks, KeymanFileTypes, KmpJsonFile, KpsFile } from "@keymanapp/common-types"; +import { CompilerCallbacks, KeymanCompiler, KeymanCompilerArtifact, KeymanCompilerArtifacts, KeymanCompilerResult, KeymanFileTypes, KmpJsonFile, KpsFile } from "@keymanapp/common-types"; import KEYMAN_VERSION from "@keymanapp/keyman-version"; -import { KmpCompiler } from "./kmp-compiler.js"; +import { KmpCompiler, KmpCompilerOptions } from "./kmp-compiler.js"; import { CompilerMessages } from "./messages.js"; const SETUP_INF_FILENAME = 'setup.inf'; @@ -30,15 +30,33 @@ export interface WindowsPackageInstallerSources { startWithConfiguration: boolean; }; -export class WindowsPackageInstallerCompiler { - private kmpCompiler: KmpCompiler; +export interface WindowsPackageInstallerCompilerOptions extends KmpCompilerOptions { + sources: WindowsPackageInstallerSources; +} - constructor(private callbacks: CompilerCallbacks) { - this.kmpCompiler = new KmpCompiler(this.callbacks); +export interface WindowsPackageInstallerCompilerArtifacts extends KeymanCompilerArtifacts { + exe: KeymanCompilerArtifact; +}; + +export interface WindowsPackageInstallerCompilerResult extends KeymanCompilerResult { + artifacts: WindowsPackageInstallerCompilerArtifacts; +}; + +export class WindowsPackageInstallerCompiler implements KeymanCompiler { + private kmpCompiler: KmpCompiler; + private callbacks: CompilerCallbacks; + private options: WindowsPackageInstallerCompilerOptions; + + async init(callbacks: CompilerCallbacks, options: WindowsPackageInstallerCompilerOptions): Promise { + this.callbacks = callbacks; + this.options = {...options}; + this.kmpCompiler = new KmpCompiler(); + return await this.kmpCompiler.init(callbacks, options); } - public async compile(kpsFilename: string, sources: WindowsPackageInstallerSources): Promise { - const kps = this.kmpCompiler.loadKpsFile(kpsFilename); + public async run(inputFilename: string, outputFilename?: string): Promise { + const sources = this.options.sources; + const kps = this.kmpCompiler.loadKpsFile(inputFilename); if(!kps) { // errors will already have been reported by loadKpsFile return null; @@ -64,7 +82,7 @@ export class WindowsPackageInstallerCompiler { // Nor do we use the MSIOptions field. // Build the zip - const zipBuffer = await this.buildZip(kps, kpsFilename, sources); + const zipBuffer = await this.buildZip(kps, inputFilename, sources); if(!zipBuffer) { // Error messages already reported by buildZip return null; @@ -72,7 +90,22 @@ export class WindowsPackageInstallerCompiler { // Build the sfx const sfxBuffer = this.buildSfx(zipBuffer, sources); - return sfxBuffer; + + const result: WindowsPackageInstallerCompilerResult = { + artifacts: { + exe: { + data: sfxBuffer, + filename: outputFilename ?? inputFilename.replace(/\.kps$/, '.exe') + } + } + }; + + return result; + } + + public async write(artifacts: WindowsPackageInstallerCompilerArtifacts): Promise { + this.callbacks.fs.writeFileSync(artifacts.exe.filename, artifacts.exe.data); + return true; } private async buildZip(kps: KpsFile.KpsFile, kpsFilename: string, sources: WindowsPackageInstallerSources): Promise { diff --git a/developer/src/kmc-package/test/fixtures/invalid/example.qaa.sencoten.model.js b/developer/src/kmc-package/test/fixtures/invalid/example.qaa.sencoten.model.js new file mode 100644 index 0000000000..2f309acbc0 --- /dev/null +++ b/developer/src/kmc-package/test/fixtures/invalid/example.qaa.sencoten.model.js @@ -0,0 +1 @@ +// dummy file for unit tests \ No newline at end of file diff --git a/developer/src/kmc-package/test/fixtures/invalid/keyman.exe b/developer/src/kmc-package/test/fixtures/invalid/keyman.exe new file mode 100644 index 0000000000..789aa9641a --- /dev/null +++ b/developer/src/kmc-package/test/fixtures/invalid/keyman.exe @@ -0,0 +1 @@ +This is a dummy file for testing the unit tests \ No newline at end of file diff --git a/developer/src/kmc-package/test/fixtures/invalid/khmer_angkor.docx b/developer/src/kmc-package/test/fixtures/invalid/khmer_angkor.docx new file mode 100644 index 0000000000..0f8c58ac58 --- /dev/null +++ b/developer/src/kmc-package/test/fixtures/invalid/khmer_angkor.docx @@ -0,0 +1 @@ +This is a sample text file \ No newline at end of file diff --git a/developer/src/kmc-package/test/test-messages.ts b/developer/src/kmc-package/test/test-messages.ts index 24fb658b59..d7a7f0175e 100644 --- a/developer/src/kmc-package/test/test-messages.ts +++ b/developer/src/kmc-package/test/test-messages.ts @@ -4,7 +4,6 @@ import { TestCompilerCallbacks, verifyCompilerMessagesObject } from '@keymanapp/ import { CompilerMessages } from '../src/compiler/messages.js'; import { makePathToFixture } from './helpers/index.js'; import { KmpCompiler } from '../src/compiler/kmp-compiler.js'; -import { PackageValidation } from '../src/compiler/package-validation.js'; import { CompilerErrorNamespace, CompilerOptions } from '@keymanapp/common-types'; const debug = false; @@ -20,24 +19,16 @@ describe('CompilerMessages', function () { // Message tests // - function testForMessage(context: Mocha.Context, fixture: string[], messageId?: number, options?: CompilerOptions) { + async function testForMessage(context: Mocha.Context, fixture: string[], messageId?: number, options?: CompilerOptions) { context.timeout(10000); callbacks.clear(); const kpsPath = makePathToFixture(...fixture); - const kmpCompiler = new KmpCompiler(callbacks); + const kmpCompiler = new KmpCompiler(); + assert.isTrue(await kmpCompiler.init(callbacks, options ?? {})); - let kmpJson = kmpCompiler.transformKpsToKmpObject(kpsPath); - if(kmpJson && callbacks.messages.length == 0) { - const validator = new PackageValidation(callbacks, options ?? {}); - validator.validate(kpsPath, kmpJson); // we'll ignore return value and rely on the messages - } - - if(kmpJson && callbacks.messages.length == 0) { - // We'll try building the package if we have not yet received any messages - kmpCompiler.buildKmpFile(kpsPath, kmpJson) - } + await kmpCompiler.run(kpsPath); if(debug) callbacks.printMessages(); @@ -52,106 +43,106 @@ describe('CompilerMessages', function () { // WARN_FileIsNotABinaryKvkFile it('should generate WARN_FileIsNotABinaryKvkFile if a non-binary kvk file is included', async function() { - testForMessage(this, ['xml_kvk_file', 'source', 'xml_kvk_file.kps'], CompilerMessages.WARN_FileIsNotABinaryKvkFile); + await testForMessage(this, ['xml_kvk_file', 'source', 'xml_kvk_file.kps'], CompilerMessages.WARN_FileIsNotABinaryKvkFile); }); it('should not warn if a binary kvk file is included', async function() { - testForMessage(this, ['binary_kvk_file', 'source', 'binary_kvk_file.kps']); + await testForMessage(this, ['binary_kvk_file', 'source', 'binary_kvk_file.kps']); }); // ERROR_FollowKeyboardVersionNotAllowedForModelPackages it('should generate ERROR_FollowKeyboardVersionNotAllowedForModelPackages if is set for model packages', async function() { - testForMessage(this, ['invalid', 'followkeyboardversion.qaa.sencoten.model.kps'], CompilerMessages.ERROR_FollowKeyboardVersionNotAllowedForModelPackages); + await testForMessage(this, ['invalid', 'followkeyboardversion.qaa.sencoten.model.kps'], CompilerMessages.ERROR_FollowKeyboardVersionNotAllowedForModelPackages); }); // ERROR_FollowKeyboardVersionButNoKeyboards it('should generate ERROR_FollowKeyboardVersionButNoKeyboards if is set for a package with no keyboards', async function() { - testForMessage(this, ['invalid', 'followkeyboardversion.empty.kps'], CompilerMessages.ERROR_FollowKeyboardVersionButNoKeyboards); + await testForMessage(this, ['invalid', 'followkeyboardversion.empty.kps'], CompilerMessages.ERROR_FollowKeyboardVersionButNoKeyboards); }); // ERROR_KeyboardContentFileNotFound it('should generate ERROR_KeyboardContentFileNotFound if a is listed in a package but not found in ', async function() { - testForMessage(this, ['invalid', 'keyboardcontentfilenotfound.kps'], CompilerMessages.ERROR_KeyboardContentFileNotFound); + await testForMessage(this, ['invalid', 'keyboardcontentfilenotfound.kps'], CompilerMessages.ERROR_KeyboardContentFileNotFound); }); // ERROR_KeyboardFileNotValid it('should generate ERROR_KeyboardFileNotValid if a .kmx is not valid in ', async function() { - testForMessage(this, ['invalid', 'keyboardfilenotvalid.kps'], CompilerMessages.ERROR_KeyboardFileNotValid); + await testForMessage(this, ['invalid', 'keyboardfilenotvalid.kps'], CompilerMessages.ERROR_KeyboardFileNotValid); }); // INFO_KeyboardFileHasNoKeyboardVersion it('should generate INFO_KeyboardFileHasNoKeyboardVersion if is set but keyboard has no version', async function() { - testForMessage(this, ['invalid', 'nokeyboardversion.kps'], CompilerMessages.INFO_KeyboardFileHasNoKeyboardVersion); + await testForMessage(this, ['invalid', 'nokeyboardversion.kps'], CompilerMessages.INFO_KeyboardFileHasNoKeyboardVersion); }); // ERROR_PackageCannotContainBothModelsAndKeyboards it('should generate ERROR_PackageCannotContainBothModelsAndKeyboards if package has both keyboards and models', async function() { - testForMessage(this, ['invalid', 'error_package_cannot_contain_both_models_and_keyboards.kps'], CompilerMessages.ERROR_PackageCannotContainBothModelsAndKeyboards); + await testForMessage(this, ['invalid', 'error_package_cannot_contain_both_models_and_keyboards.kps'], CompilerMessages.ERROR_PackageCannotContainBothModelsAndKeyboards); }); // HINT_PackageShouldNotRepeatLanguages (models) it('should generate HINT_PackageShouldNotRepeatLanguages if model has same language repeated', async function() { - testForMessage(this, ['invalid', 'keyman.en.hint_package_should_not_repeat_languages.model.kps'], CompilerMessages.HINT_PackageShouldNotRepeatLanguages); + await testForMessage(this, ['invalid', 'keyman.en.hint_package_should_not_repeat_languages.model.kps'], CompilerMessages.HINT_PackageShouldNotRepeatLanguages); }); // HINT_PackageShouldNotRepeatLanguages (keyboards) it('should generate HINT_PackageShouldNotRepeatLanguages if keyboard has same language repeated', async function() { - testForMessage(this, ['invalid', 'hint_package_should_not_repeat_languages.kps'], CompilerMessages.HINT_PackageShouldNotRepeatLanguages); + await testForMessage(this, ['invalid', 'hint_package_should_not_repeat_languages.kps'], CompilerMessages.HINT_PackageShouldNotRepeatLanguages); }); // WARN_PackageNameDoesNotFollowLexicalModelConventions it('should generate WARN_PackageNameDoesNotFollowLexicalModelConventions if filename has wrong conventions', async function() { - testForMessage(this, ['invalid', 'WARN_PackageNameDoesNotFollowLexicalModelConventions.kps'], CompilerMessages.WARN_PackageNameDoesNotFollowLexicalModelConventions); + await testForMessage(this, ['invalid', 'WARN_PackageNameDoesNotFollowLexicalModelConventions.kps'], CompilerMessages.WARN_PackageNameDoesNotFollowLexicalModelConventions); }); // WARN_PackageNameDoesNotFollowKeyboardConventions it('should generate WARN_PackageNameDoesNotFollowKeyboardConventions if filename has wrong conventions', async function() { - testForMessage(this, ['invalid', 'WARN_PackageNameDoesNotFollowKeyboardConventions.kps'], CompilerMessages.WARN_PackageNameDoesNotFollowKeyboardConventions); + await testForMessage(this, ['invalid', 'WARN_PackageNameDoesNotFollowKeyboardConventions.kps'], CompilerMessages.WARN_PackageNameDoesNotFollowKeyboardConventions); }); // WARN_FileInPackageDoesNotFollowFilenameConventions it('should generate WARN_FileInPackageDoesNotFollowFilenameConventions if content filename has wrong conventions', async function() { - testForMessage(this, ['invalid', 'warn_file_in_package_does_not_follow_filename_conventions.kps'], + await testForMessage(this, ['invalid', 'warn_file_in_package_does_not_follow_filename_conventions.kps'], CompilerMessages.WARN_FileInPackageDoesNotFollowFilenameConventions, {checkFilenameConventions: true}); - testForMessage(this, ['invalid', 'warn_file_in_package_does_not_follow_filename_conventions_2.kps'], + await testForMessage(this, ['invalid', 'warn_file_in_package_does_not_follow_filename_conventions_2.kps'], CompilerMessages.WARN_FileInPackageDoesNotFollowFilenameConventions, {checkFilenameConventions: true}); }); // Test the inverse -- no warning generated if checkFilenameConventions is false it('should not generate WARN_FileInPackageDoesNotFollowFilenameConventions if content filename has wrong conventions but checkFilenameConventions is false', async function() { - testForMessage(this, ['invalid', 'warn_file_in_package_does_not_follow_filename_conventions.kps'], null, {checkFilenameConventions: false}); - testForMessage(this, ['invalid', 'warn_file_in_package_does_not_follow_filename_conventions_2.kps'], null, {checkFilenameConventions: false}); + await testForMessage(this, ['invalid', 'warn_file_in_package_does_not_follow_filename_conventions.kps'], null, {checkFilenameConventions: false}); + await testForMessage(this, ['invalid', 'warn_file_in_package_does_not_follow_filename_conventions_2.kps'], null, {checkFilenameConventions: false}); }); // ERROR_PackageNameCannotBeBlank it('should generate ERROR_PackageNameCannotBeBlank if package info has empty name', async function() { - testForMessage(this, ['invalid', 'error_package_name_cannot_be_blank.kps'], CompilerMessages.ERROR_PackageNameCannotBeBlank); // blank field - testForMessage(this, ['invalid', 'error_package_name_cannot_be_blank_2.kps'], CompilerMessages.ERROR_PackageNameCannotBeBlank); // missing field + await testForMessage(this, ['invalid', 'error_package_name_cannot_be_blank.kps'], CompilerMessages.ERROR_PackageNameCannotBeBlank); // blank field + await testForMessage(this, ['invalid', 'error_package_name_cannot_be_blank_2.kps'], CompilerMessages.ERROR_PackageNameCannotBeBlank); // missing field }); // ERROR_KeyboardFileNotFound it('should generate ERROR_KeyboardFileNotFound if a is listed in a package but not found in ', async function() { - testForMessage(this, ['invalid', 'keyboardfilenotfound.kps'], CompilerMessages.ERROR_KeyboardFileNotFound); + await testForMessage(this, ['invalid', 'keyboardfilenotfound.kps'], CompilerMessages.ERROR_KeyboardFileNotFound); }); // WARN_KeyboardVersionsDoNotMatch it('should generate WARN_KeyboardVersionsDoNotMatch if two have different versions', async function() { - testForMessage(this, ['invalid', 'warn_keyboard_versions_do_not_match.kps'], CompilerMessages.WARN_KeyboardVersionsDoNotMatch); + await testForMessage(this, ['invalid', 'warn_keyboard_versions_do_not_match.kps'], CompilerMessages.WARN_KeyboardVersionsDoNotMatch); }); // ERROR_LanguageTagIsNotValid @@ -163,73 +154,73 @@ describe('CompilerMessages', function () { // HINT_LanguageTagIsNotMinimal it('should generate HINT_LanguageTagIsNotMinimal if keyboard has a non-minimal language tag', async function() { - testForMessage(this, ['invalid', 'hint_language_tag_is_not_minimal.kps'], CompilerMessages.HINT_LanguageTagIsNotMinimal); + await testForMessage(this, ['invalid', 'hint_language_tag_is_not_minimal.kps'], CompilerMessages.HINT_LanguageTagIsNotMinimal); }); // ERROR_ModelMustHaveAtLeastOneLanguage it('should generate ERROR_MustHaveAtLeastOneLanguage if model has zero language tags', async function() { - testForMessage(this, ['invalid', 'keyman.en.error_model_must_have_at_least_one_language.model.kps'], + await testForMessage(this, ['invalid', 'keyman.en.error_model_must_have_at_least_one_language.model.kps'], CompilerMessages.ERROR_ModelMustHaveAtLeastOneLanguage); }); // WARN_RedistFileShouldNotBeInPackage it('should generate WARN_RedistFileShouldNotBeInPackage if package contains a redist file', async function() { - testForMessage(this, ['invalid', 'warn_redist_file_should_not_be_in_package.kps'], + await testForMessage(this, ['invalid', 'warn_redist_file_should_not_be_in_package.kps'], CompilerMessages.WARN_RedistFileShouldNotBeInPackage); }); // WARN_DocFileDangerous it('should generate WARN_DocFileDangerous if package contains a .doc file', async function() { - testForMessage(this, ['invalid', 'warn_doc_file_dangerous.kps'], + await testForMessage(this, ['invalid', 'warn_doc_file_dangerous.kps'], CompilerMessages.WARN_DocFileDangerous); }); // ERROR_PackageMustContainAPackageOrAKeyboard it('should generate ERROR_PackageMustContainAModelOrAKeyboard if package contains no keyboard or model', async function() { - testForMessage(this, ['invalid', 'error_package_must_contain_a_model_or_a_keyboard.kps'], + await testForMessage(this, ['invalid', 'error_package_must_contain_a_model_or_a_keyboard.kps'], CompilerMessages.ERROR_PackageMustContainAModelOrAKeyboard); }); // WARN_JsKeyboardFileIsMissing it('should generate WARN_JsKeyboardFileIsMissing if package is missing corresponding .js for a touch .kmx', async function() { - testForMessage(this, ['invalid', 'warn_js_keyboard_file_is_missing.kps'], + await testForMessage(this, ['invalid', 'warn_js_keyboard_file_is_missing.kps'], CompilerMessages.WARN_JsKeyboardFileIsMissing); }); // WARN_KeyboardShouldHaveAtLeastOneLanguage it('should generate WARN_KeyboardShouldHaveAtLeastOneLanguage if keyboard has zero language tags', async function() { - testForMessage(this, ['invalid', 'warn_keyboard_should_have_at_least_one_language.kps'], + await testForMessage(this, ['invalid', 'warn_keyboard_should_have_at_least_one_language.kps'], CompilerMessages.WARN_KeyboardShouldHaveAtLeastOneLanguage); }); // HINT_JsKeyboardFileHasNoTouchTargets it('should generate HINT_JsKeyboardFileHasNoTouchTargets if keyboard has no touch targets', async function() { - testForMessage(this, ['invalid', 'hint_js_keyboard_file_has_no_touch_targets.kps'], + await testForMessage(this, ['invalid', 'hint_js_keyboard_file_has_no_touch_targets.kps'], CompilerMessages.HINT_JsKeyboardFileHasNoTouchTargets); }); it('should not generate HINT_JsKeyboardFileHasNoTouchTargets if keyboard has a touch target', async function() { - testForMessage(this, ['khmer_angkor', 'source', 'khmer_angkor.kps'], null); + await testForMessage(this, ['khmer_angkor', 'source', 'khmer_angkor.kps'], null); }); // HINT_PackageContainsSourceFile it('should generate HINT_PackageContainsSourceFile if package contains a source file', async function() { - testForMessage(this, ['invalid', 'hint_source_file_should_not_be_in_package.kps'], + await testForMessage(this, ['invalid', 'hint_source_file_should_not_be_in_package.kps'], CompilerMessages.HINT_PackageContainsSourceFile); }); // ERROR_InvalidPackageFile it('should generate ERROR_InvalidPackageFile if package source file contains invalid XML', async function() { - testForMessage(this, ['invalid', 'error_invalid_package_file.kps'], + await testForMessage(this, ['invalid', 'error_invalid_package_file.kps'], CompilerMessages.ERROR_InvalidPackageFile); }); diff --git a/developer/src/kmc-package/test/test-package-compiler.ts b/developer/src/kmc-package/test/test-package-compiler.ts index 433d730c18..1398befca2 100644 --- a/developer/src/kmc-package/test/test-package-compiler.ts +++ b/developer/src/kmc-package/test/test-package-compiler.ts @@ -20,9 +20,13 @@ describe('KmpCompiler', function () { 'example.qaa.sencoten', 'withfolders.qaa.sencoten', ]; - const callbacks = new TestCompilerCallbacks(); - let kmpCompiler = new KmpCompiler(callbacks); + let kmpCompiler: KmpCompiler = null; + + this.beforeAll(async function() { + kmpCompiler = new KmpCompiler(); + assert.isTrue(await kmpCompiler.init(callbacks, null)); + }); for (let modelID of MODELS) { const kpsPath = modelID.includes('withfolders') ? @@ -89,7 +93,9 @@ describe('KmpCompiler', function () { const kpsPath = makePathToFixture('khmer_angkor', 'source', 'khmer_angkor.kps'); const kmpJsonRefPath = makePathToFixture('khmer_angkor', 'ref', 'kmp.json'); - const kmpCompiler = new KmpCompiler(callbacks); + const kmpCompiler = new KmpCompiler(); + assert.isTrue(await kmpCompiler.init(callbacks, null)); + const kmpJsonFixture: KmpJsonFile.KmpJsonFile = JSON.parse(fs.readFileSync(kmpJsonRefPath, 'utf-8')); // We override the fixture version so that we can compare with the compiler output @@ -193,7 +199,8 @@ describe('KmpCompiler', function () { callbacks.clear(); const kpsPath = makePathToFixture('absolute_path', 'source', 'absolute_path.kps'); - const kmpCompiler = new KmpCompiler(callbacks); + const kmpCompiler = new KmpCompiler(); + assert.isTrue(await kmpCompiler.init(callbacks, null)); let kmpJson: KmpJsonFile.KmpJsonFile = null; diff --git a/developer/src/kmc-package/test/test-versioning.ts b/developer/src/kmc-package/test/test-versioning.ts index e9771a369a..98d00b853c 100644 --- a/developer/src/kmc-package/test/test-versioning.ts +++ b/developer/src/kmc-package/test/test-versioning.ts @@ -33,7 +33,9 @@ describe('package versioning', function () { for(const [ caseTitle, filename ] of cases) { it(caseTitle, async function () { const callbacks = new TestCompilerCallbacks(); - const kmpCompiler = new KmpCompiler(callbacks); + const kmpCompiler = new KmpCompiler(); + assert.isTrue(await kmpCompiler.init(callbacks, null)); + const kpsPath = makePathToFixture('versioning', filename); const kmpJson: KmpJsonFile.KmpJsonFile = kmpCompiler.transformKpsToKmpObject(kpsPath); assert.isTrue(kmpJson !== null); diff --git a/developer/src/kmc-package/test/test-windows-package-installer-compiler.ts b/developer/src/kmc-package/test/test-windows-package-installer-compiler.ts index 8ed2fee0e7..e1fde65647 100644 --- a/developer/src/kmc-package/test/test-windows-package-installer-compiler.ts +++ b/developer/src/kmc-package/test/test-windows-package-installer-compiler.ts @@ -12,9 +12,6 @@ describe('WindowsPackageInstallerCompiler', function () { it(`should build an SFX archive`, async function () { this.timeout(10000); // this test can take a little while to run - const callbacks = new TestCompilerCallbacks(); - let compiler = new WindowsPackageInstallerCompiler(callbacks); - const kpsPath = makePathToFixture('khmer_angkor', 'source', 'khmer_angkor.kps'); const sources: WindowsPackageInstallerSources = { licenseFilename: makePathToFixture('windows-installer', 'license.txt'), @@ -25,12 +22,19 @@ describe('WindowsPackageInstallerCompiler', function () { appName: 'Testing', }; - const sfxBuffer = await compiler.compile(kpsPath, sources); + const callbacks = new TestCompilerCallbacks(); + let compiler = new WindowsPackageInstallerCompiler(); + assert.isTrue(await compiler.init(callbacks, {sources})); + + const result = await compiler.run(kpsPath, null); + assert.isNotNull(result); // This returns a buffer with a SFX loader and a zip suffix. For the sake of repository size // we actually provide a stub SFX loader and a stub MSI file, which is enough to verify that // the compiler is generating what it thinks is a valid file. + const sfxBuffer = result.artifacts.exe.data; + const zip = JSZip(); // Check that file.kmp contains just 3 files - setup.inf, keymandesktop.msi, and khmer_angkor.kmp, diff --git a/developer/src/kmc/src/commands/analyze.ts b/developer/src/kmc/src/commands/analyze.ts index eb2feb528a..15c8ac3f39 100644 --- a/developer/src/kmc/src/commands/analyze.ts +++ b/developer/src/kmc/src/commands/analyze.ts @@ -105,7 +105,7 @@ async function analyzeOskCharUse(callbacks: CompilerCallbacks, filenames: string async function analyzeOskRewritePua(callbacks: CompilerCallbacks, filenames: string[], options: AnalysisActivityOptions) { const analyzer = new AnalyzeOskRewritePua(callbacks); - const mapping: any = JSON.parse(callbacks.fs.readFileSync(options.mappingFile, 'UTF-8')); + const mapping: any = JSON.parse(fs.readFileSync(options.mappingFile, 'utf-8')); return await runOnFiles(callbacks, filenames, async (filename: string): Promise => { if(!await analyzer.analyze(filename, mapping)) { diff --git a/developer/src/kmc/src/commands/build.ts b/developer/src/kmc/src/commands/build.ts index 0d091680b7..21aa6fdc3b 100644 --- a/developer/src/kmc/src/commands/build.ts +++ b/developer/src/kmc/src/commands/build.ts @@ -19,7 +19,6 @@ function commandOptionsToCompilerOptions(options: any): ExtendedCompilerOptions // CompilerOptions properties... return { // CompilerBaseOptions - outFile: options.outFile, logLevel: options.logLevel, logFormat: options.logFormat, color: options.color, @@ -66,7 +65,8 @@ File lists can be referenced with @filelist.txt. If no input file is supplied, kmc will build the current folder.`) .action(async (filenames: string[], _options: any, commander: any) => { - const options = commandOptionsToCompilerOptions(commander.optsWithGlobals()); + const commanderOptions/*:{TODO?} CommandLineCompilerOptions*/ = commander.optsWithGlobals(); + const options = commandOptionsToCompilerOptions(commanderOptions); const callbacks = new NodeCompilerCallbacks(options); if(!filenames.length) { @@ -75,12 +75,17 @@ If no input file is supplied, kmc will build the current folder.`) filenames.push('.'); } + if(filenames.length > 1 && commanderOptions.outFile) { + // -o can only be specified with a single input file + callbacks.reportMessage(InfrastructureMessages.Error_OutFileCanOnlyBeSpecifiedWithSingleInfile()); + } + if(!expandFileLists(filenames, callbacks)) { process.exit(1); } for(let filename of filenames) { - if(!await build(filename, callbacks, options)) { + if(!await build(filename, commanderOptions.outFile, callbacks, options)) { // Once a file fails to build, we bail on subsequent builds process.exit(1); } @@ -105,7 +110,7 @@ If no input file is supplied, kmc will build the current folder.`) .action(buildWindowsPackageInstaller); } -async function build(filename: string, parentCallbacks: NodeCompilerCallbacks, options: CompilerOptions): Promise { +async function build(filename: string, outfile: string, parentCallbacks: NodeCompilerCallbacks, options: CompilerOptions): Promise { try { // TEST: allow command-line simulation of infrastructure fatal errors, and // also for unit tests @@ -151,7 +156,7 @@ async function build(filename: string, parentCallbacks: NodeCompilerCallbacks, o const callbacks = new CompilerFileCallbacks(buildFilename, options, parentCallbacks); callbacks.reportMessage(InfrastructureMessages.Info_BuildingFile({filename:buildFilename, relativeFilename})); - let result = await builder.build(filename, callbacks, options); + let result = await builder.build(filename, outfile, callbacks, options); result = result && !callbacks.hasFailureMessage(); if(result) { callbacks.reportMessage(builder instanceof BuildProject diff --git a/developer/src/kmc/src/commands/buildClasses/BuildActivity.ts b/developer/src/kmc/src/commands/buildClasses/BuildActivity.ts index dae545366f..864611c26d 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildActivity.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildActivity.ts @@ -1,15 +1,42 @@ -import { CompilerCallbacks, CompilerOptions, KeymanFileTypes } from "@keymanapp/common-types"; -import { escapeRegExp } from "../../util/escapeRegExp.js"; +import * as fs from 'fs'; +import * as path from 'path'; +import { CompilerCallbacks, CompilerOptions, KeymanCompiler, KeymanFileTypes } from "@keymanapp/common-types"; +import { InfrastructureMessages } from '../../messages/infrastructureMessages.js'; export abstract class BuildActivity { public abstract get name(): string; public abstract get sourceExtension(): KeymanFileTypes.Source; public abstract get compiledExtension(): KeymanFileTypes.Binary; public abstract get description(): string; - public abstract build(infile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise; - protected getOutputFilename(infile: string, options: CompilerOptions): string { - return options.outFile ? - options.outFile : - infile.replace(new RegExp(escapeRegExp(this.sourceExtension), "g"), this.compiledExtension); + public abstract build(infile: string, outfile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise; + + protected async runCompiler(compiler: KeymanCompiler, infile: string, outfile: string, callbacks: CompilerCallbacks, options: T): Promise { + if(!await compiler.init(callbacks, options)) { + return false; + } + + const result = await compiler.run(infile, outfile); + if(!result) { + return false; + } + + if(!this.createOutputFolder(outfile ?? infile, callbacks)) { + return false; + } + + return await compiler.write(result.artifacts); + } + + private createOutputFolder(targetFilename: string, callbacks: CompilerCallbacks): boolean { + const targetFolder = path.dirname(targetFilename); + + try { + fs.mkdirSync(targetFolder, {recursive: true}); + } catch(e) { + callbacks.reportMessage(InfrastructureMessages.Error_CannotCreateFolder({folderName:targetFolder, e})); + return false; + } + + return true; } }; \ No newline at end of file diff --git a/developer/src/kmc/src/commands/buildClasses/BuildKeyboardInfo.ts b/developer/src/kmc/src/commands/buildClasses/BuildKeyboardInfo.ts index 936c358564..605870741e 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildKeyboardInfo.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildKeyboardInfo.ts @@ -13,7 +13,7 @@ export class BuildKeyboardInfo extends BuildActivity { public get sourceExtension(): KeymanFileTypes.Source { return KeymanFileTypes.Source.Project; } public get compiledExtension(): KeymanFileTypes.Binary { return KeymanFileTypes.Binary.KeyboardInfo; } public get description(): string { return 'Build a keyboard metadata file'; } - public async build(infile: string, callbacks: CompilerCallbacks, options: ExtendedCompilerOptions): Promise { + public async build(infile: string, _outfile: string, callbacks: CompilerCallbacks, options: ExtendedCompilerOptions): Promise { if(!KeymanFileTypes.filenameIs(infile, KeymanFileTypes.Source.Project)) { // Even if the project file does not exist, we use its name as our reference // in order to avoid ambiguity @@ -35,29 +35,17 @@ export class BuildKeyboardInfo extends BuildActivity { const keyboard = project.files.find(file => file.fileType == KeymanFileTypes.Source.KeymanKeyboard); const jsFilename = keyboard ? project.resolveOutputFilePath(keyboard, KeymanFileTypes.Source.KeymanKeyboard, KeymanFileTypes.Binary.WebKeyboard) : null; const lastCommitDate = getLastGitCommitDate(project.projectPath); - - const compiler = new KeyboardInfoCompiler(callbacks); - const data = await compiler.writeKeyboardInfoFile({ + const sources = { kmpFilename: project.resolveOutputFilePath(kps, KeymanFileTypes.Source.Package, KeymanFileTypes.Binary.Package), kpsFilename: project.resolveInputFilePath(kps), jsFilename: jsFilename && fs.existsSync(jsFilename) ? jsFilename : undefined, sourcePath: calculateSourcePath(infile), lastCommitDate, forPublishing: !!options.forPublishing, - }); - - if(data == null) { - // Error messages have already been emitted by KeyboardInfoCompiler - return false; - } - + }; + // Note: should we always ignore the passed-in output filename for .keyboard_info? const outputFilename = project.getOutputFilePath(KeymanFileTypes.Binary.KeyboardInfo); - - fs.writeFileSync( - outputFilename, - data - ); - - return true; + const compiler = new KeyboardInfoCompiler(); + return await super.runCompiler(compiler, infile, outputFilename, callbacks, {...options, sources}); } } diff --git a/developer/src/kmc/src/commands/buildClasses/BuildKmnKeyboard.ts b/developer/src/kmc/src/commands/buildClasses/BuildKmnKeyboard.ts index 61dd2f97a6..e540aba451 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildKmnKeyboard.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildKmnKeyboard.ts @@ -3,33 +3,21 @@ import { platform } from 'os'; import { KmnCompiler } from '@keymanapp/kmc-kmn'; import { CompilerOptions, CompilerCallbacks, KeymanFileTypes } from '@keymanapp/common-types'; import { BuildActivity } from './BuildActivity.js'; -import * as fs from 'fs'; -import { InfrastructureMessages } from '../../messages/infrastructureMessages.js'; export class BuildKmnKeyboard extends BuildActivity { public get name(): string { return 'Keyman keyboard'; } public get sourceExtension(): KeymanFileTypes.Source { return KeymanFileTypes.Source.KeymanKeyboard; } public get compiledExtension(): KeymanFileTypes.Binary { return KeymanFileTypes.Binary.Keyboard; } public get description(): string { return 'Build a Keyman keyboard'; } - public async build(infile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise { - let compiler = new KmnCompiler(); - if(!await compiler.init(callbacks)) { - return false; + public async build(infile: string, outfile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise { + // We need to resolve paths to absolute paths before calling kmc-kmn + infile = getPosixAbsolutePath(infile); + if(outfile) { + outfile = getPosixAbsolutePath(outfile); } - // We need to resolve paths to absolute paths before calling kmc-kmn - if(options.outFile) { - options.outFile = getPosixAbsolutePath(options.outFile); - const folderName = path.dirname(options.outFile); - try { - fs.mkdirSync(folderName, {recursive: true}); - } catch(e) { - callbacks.reportMessage(InfrastructureMessages.Error_CannotCreateFolder({folderName, e})); - return false; - } - } - infile = getPosixAbsolutePath(infile); - return compiler.run(infile, options); + const compiler = new KmnCompiler(); + return await super.runCompiler(compiler, infile, outfile, callbacks, options); } } diff --git a/developer/src/kmc/src/commands/buildClasses/BuildLdmlKeyboard.ts b/developer/src/kmc/src/commands/buildClasses/BuildLdmlKeyboard.ts index 551ce66fa4..0e0cfd9766 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildLdmlKeyboard.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildLdmlKeyboard.ts @@ -1,101 +1,19 @@ -import * as path from 'path'; -import * as fs from 'fs'; import * as kmcLdml from '@keymanapp/kmc-ldml'; -import { KvkFileWriter, CompilerCallbacks, LDMLKeyboardXMLSourceFileReader, CompilerOptions, defaultCompilerOptions, KeymanFileTypes } from '@keymanapp/common-types'; +import { CompilerCallbacks, LDMLKeyboardXMLSourceFileReader, CompilerOptions, KeymanFileTypes } from '@keymanapp/common-types'; import { BuildActivity } from './BuildActivity.js'; import { fileURLToPath } from 'url'; -import { InfrastructureMessages } from '../../messages/infrastructureMessages.js'; export class BuildLdmlKeyboard extends BuildActivity { public get name(): string { return 'LDML keyboard'; } public get sourceExtension(): KeymanFileTypes.Source { return KeymanFileTypes.Source.LdmlKeyboard; } public get compiledExtension(): KeymanFileTypes.Binary { return KeymanFileTypes.Binary.Keyboard; } public get description(): string { return 'Build a LDML keyboard'; } - public async build(infile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise { + public async build(infile: string, outfile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise { // TODO-LDML: consider hardware vs touch -- touch-only layout will not have a .kvk - // Compile: - let [kmx,kvk,kmw] = await buildLdmlKeyboardToMemory(infile, callbacks, options); - // Output: - - const fileBaseName = options.outFile ?? infile; - const outFileBase = path.basename(fileBaseName, path.extname(fileBaseName)); - const outFileDir = path.dirname(fileBaseName); - - try { - fs.mkdirSync(outFileDir, {recursive: true}); - } catch(e) { - callbacks.reportMessage(InfrastructureMessages.Error_CannotCreateFolder({folderName:outFileDir, e})); - return false; - } - - if(kmx && kvk) { - const outFileKmx = path.join(outFileDir, outFileBase + KeymanFileTypes.Binary.Keyboard); - // TODO: console needs to be replaced with InfrastructureMessages - console.log(`Writing compiled keyboard to ${outFileKmx}`); - fs.writeFileSync(outFileKmx, kmx); - - const outFileKvk = path.join(outFileDir, outFileBase + KeymanFileTypes.Binary.VisualKeyboard); - // TODO: console needs to be replaced with InfrastructureMessages - console.log(`Writing compiled visual keyboard to ${outFileKvk}`); - fs.writeFileSync(outFileKvk, kvk); - } else { - // TODO: console needs to be replaced with InfrastructureMessages - console.error(`An error occurred compiling ${infile}`); - return false; - } - - if(kmw) { - const outFileKmw = path.join(outFileDir, outFileBase + KeymanFileTypes.Binary.WebKeyboard); - // TODO: console needs to be replaced with InfrastructureMessages - console.log(`Writing compiled js keyboard to ${outFileKmw}`); - fs.writeFileSync(outFileKmw, kmw); - } - - return true; - } -} - -async function buildLdmlKeyboardToMemory(inputFilename: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise<[Uint8Array, Uint8Array, Uint8Array]> { - let compilerOptions: kmcLdml.LdmlCompilerOptions = { - ...defaultCompilerOptions, - ...options, - readerOptions: { + const ldmlCompilerOptions: kmcLdml.LdmlCompilerOptions = {...options, readerOptions: { importsPath: fileURLToPath(new URL(...LDMLKeyboardXMLSourceFileReader.defaultImportsURL)) - } - }; - - const k = new kmcLdml.LdmlKeyboardCompiler(callbacks, compilerOptions); - let source = k.load(inputFilename); - if (!source) { - return [null, null, null]; + }}; + const compiler = new kmcLdml.LdmlKeyboardCompiler(); + return await super.runCompiler(compiler, infile, outfile, callbacks, ldmlCompilerOptions); } - let kmx = await k.compile(source); - if (!kmx) { - return [null, null, null]; - } - - // In order for the KMX file to be loaded by non-KMXPlus components, it is helpful - // to duplicate some of the metadata - kmcLdml.KMXPlusMetadataCompiler.addKmxMetadata(kmx.kmxplus, kmx.keyboard, compilerOptions); - - // Use the builder to generate the binary output file - const builder = new kmcLdml.KMXBuilder(kmx, options.saveDebug); - const kmx_binary = builder.compile(); - - const vkcompiler = new kmcLdml.LdmlKeyboardVisualKeyboardCompiler(callbacks); - const vk = vkcompiler.compile(source); - const writer = new KvkFileWriter(); - const kvk_binary = writer.write(vk); - - // Note: we could have a step of generating source files here - // KvksFileWriter()... - // const tlcompiler = new kmc.TouchLayoutCompiler(); - // const tl = tlcompiler.compile(source); - // const tlwriter = new TouchLayoutFileWriter(); - const kmwcompiler = new kmcLdml.LdmlKeyboardKeymanWebCompiler(callbacks, compilerOptions); - const kmw_string = kmwcompiler.compile(inputFilename, source); - const encoder = new TextEncoder(); - const kmw_binary = encoder.encode(kmw_string); - - return [kmx_binary, kvk_binary, kmw_binary]; } diff --git a/developer/src/kmc/src/commands/buildClasses/BuildModel.ts b/developer/src/kmc/src/commands/buildClasses/BuildModel.ts index e69aeca780..b2afc59294 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildModel.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildModel.ts @@ -1,6 +1,5 @@ -import * as fs from 'fs'; import { BuildActivity } from './BuildActivity.js'; -import { compileModel } from '@keymanapp/kmc-model'; +import { LexicalModelCompiler } from '@keymanapp/kmc-model'; import { CompilerCallbacks, CompilerOptions, KeymanFileTypes } from '@keymanapp/common-types'; export class BuildModel extends BuildActivity { @@ -8,26 +7,8 @@ export class BuildModel extends BuildActivity { public get sourceExtension(): KeymanFileTypes.Source { return KeymanFileTypes.Source.Model; } public get compiledExtension(): KeymanFileTypes.Binary { return KeymanFileTypes.Binary.Model; } public get description(): string { return 'Build a lexical model'; } - public async build(infile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise { - let outputFilename: string = this.getOutputFilename(infile, options); - let code = null; - - // Compile: - try { - code = compileModel(infile, callbacks); - } catch(e) { - console.error(e); - return false; - } - - if(!code) { - console.error('Compilation failed.') - return false; - } - - // Output: - fs.writeFileSync(outputFilename, code, 'utf8'); - - return true; + public async build(infile: string, outfile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise { + const compiler = new LexicalModelCompiler(); + return await super.runCompiler(compiler, infile, outfile, callbacks, options); } } \ No newline at end of file diff --git a/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts b/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts index bddfcc38db..470c90639d 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts @@ -1,4 +1,4 @@ -import * as fs from 'fs'; +import * as path from 'path'; import { BuildActivity } from './BuildActivity.js'; import { CompilerCallbacks, KeymanFileTypes } from '@keymanapp/common-types'; import { ModelInfoCompiler } from '@keymanapp/kmc-model-info'; @@ -25,7 +25,7 @@ export class BuildModelInfo extends BuildActivity { * @param options * @returns */ - public async build(infile: string, callbacks: CompilerCallbacks, options: ExtendedCompilerOptions): Promise { + public async build(infile: string, _outfile: string, callbacks: CompilerCallbacks, options: ExtendedCompilerOptions): Promise { if(!KeymanFileTypes.filenameIs(infile, KeymanFileTypes.Source.Project)) { // Even if the project file does not exist, we use its name as our reference // in order to avoid ambiguity @@ -50,7 +50,12 @@ export class BuildModelInfo extends BuildActivity { return false; } - let kmpCompiler = new KmpCompiler(callbacks); + let kmpCompiler = new KmpCompiler(); + if(!await kmpCompiler.init(callbacks, options)) { + // Errors will have been emitted by KmpCompiler + return false; + } + let kmpJsonData = kmpCompiler.transformKpsToKmpObject(project.resolveInputFilePath(kps)); if(!kmpJsonData) { // Errors will have been emitted by KmpCompiler @@ -58,9 +63,8 @@ export class BuildModelInfo extends BuildActivity { } const lastCommitDate = getLastGitCommitDate(project.projectPath); - const compiler = new ModelInfoCompiler(callbacks); - const data = compiler.writeModelMetadataFile({ - model_id: callbacks.path.basename(project.projectPath, KeymanFileTypes.Source.Project), + const sources = { + model_id: path.basename(project.projectPath, KeymanFileTypes.Source.Project), kmpJsonData, sourcePath: calculateSourcePath(infile), modelFileName: project.resolveOutputFilePath(model, KeymanFileTypes.Source.Model, KeymanFileTypes.Binary.Model), @@ -68,18 +72,12 @@ export class BuildModelInfo extends BuildActivity { kpsFilename: project.resolveInputFilePath(kps), lastCommitDate, forPublishing: !!options.forPublishing, - }); + }; - if(data == null) { - // Error messages have already been emitted by writeModelMetadataFile - return false; - } + // Note: should we always ignore the passed-in output filename for .model_info? + const outputFilename = project.getOutputFilePath(KeymanFileTypes.Binary.ModelInfo); - fs.writeFileSync( - project.getOutputFilePath(KeymanFileTypes.Binary.ModelInfo), - data - ); - - return true; + const compiler = new ModelInfoCompiler(); + return await super.runCompiler(compiler, infile, outputFilename, callbacks, {...options, sources}); } } \ No newline at end of file diff --git a/developer/src/kmc/src/commands/buildClasses/BuildPackage.ts b/developer/src/kmc/src/commands/buildClasses/BuildPackage.ts index 8adcd4a92c..1bb26b3b7f 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildPackage.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildPackage.ts @@ -1,46 +1,14 @@ -import * as fs from 'fs'; import { BuildActivity } from './BuildActivity.js'; import { CompilerCallbacks, CompilerOptions, KeymanFileTypes } from '@keymanapp/common-types'; -import { KmpCompiler, PackageValidation } from '@keymanapp/kmc-package'; +import { KmpCompiler } from '@keymanapp/kmc-package'; export class BuildPackage extends BuildActivity { public get name(): string { return 'Package'; } public get sourceExtension(): KeymanFileTypes.Source { return KeymanFileTypes.Source.Package; } public get compiledExtension(): KeymanFileTypes.Binary { return KeymanFileTypes.Binary.Package; } public get description(): string { return 'Build a Keyman package'; } - public async build(infile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise { - - const outfile = this.getOutputFilename(infile, options); - - // - // Load .kps source data - // - - const kmpCompiler = new KmpCompiler(callbacks); - const kmpJsonData = kmpCompiler.transformKpsToKmpObject(infile); - if(!kmpJsonData) { - return false; - } - - // - // Validate the package file - // - - const validation = new PackageValidation(callbacks, options); - if(!validation.validate(infile, kmpJsonData)) { - return false; - } - - // - // Build the .kmp package file - // - - const data = await kmpCompiler.buildKmpFile(infile, kmpJsonData); - if(!data) { - return false; - } - - fs.writeFileSync(outfile, data, 'binary'); - return true; + public async build(infile: string, outfile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise { + const compiler = new KmpCompiler(); + return await super.runCompiler(compiler, infile, outfile, callbacks, options); } } diff --git a/developer/src/kmc/src/commands/buildClasses/BuildProject.ts b/developer/src/kmc/src/commands/buildClasses/BuildProject.ts index 38dd2ae6da..eece204de3 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildProject.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildProject.ts @@ -12,7 +12,12 @@ export class BuildProject extends BuildActivity { public get sourceExtension(): KeymanFileTypes.Source { return KeymanFileTypes.Source.Project; } public get compiledExtension(): KeymanFileTypes.Binary { return null; } public get description(): string { return 'Build a keyboard or lexical model project'; } - public async build(infile: string, callbacks: CompilerCallbacks, options: ExtendedCompilerOptions): Promise { + public async build(infile: string, outfile: string, callbacks: CompilerCallbacks, options: ExtendedCompilerOptions): Promise { + if(outfile) { + callbacks.reportMessage(InfrastructureMessages.Error_OutFileNotValidForProjects()); + return false; + } + let builder = new ProjectBuilder(infile, callbacks, options); return builder.run(); } @@ -31,11 +36,6 @@ class ProjectBuilder { } async run(): Promise { - if(this.options.outFile) { - this.callbacks.reportMessage(InfrastructureMessages.Error_OutFileNotValidForProjects()); - return false; - } - this.project = loadProject(this.infile, this.callbacks); if(!this.project) { return false; @@ -102,7 +102,7 @@ class ProjectBuilder { async buildTarget(file: KeymanDeveloperProjectFile, activity: BuildActivity): Promise { const options = {...this.options}; - options.outFile = this.project.resolveOutputFilePath(file, activity.sourceExtension, activity.compiledExtension); + const outfile = this.project.resolveOutputFilePath(file, activity.sourceExtension, activity.compiledExtension); options.checkFilenameConventions = this.project.options.checkFilenameConventions ?? this.options.checkFilenameConventions; const infile = this.project.resolveInputFilePath(file); @@ -110,9 +110,9 @@ class ProjectBuilder { const callbacks = new CompilerFileCallbacks(buildFilename, options, this.callbacks); callbacks.reportMessage(InfrastructureMessages.Info_BuildingFile({filename: infile, relativeFilename:buildFilename})); - fs.mkdirSync(path.dirname(options.outFile), {recursive:true}); + fs.mkdirSync(path.dirname(outfile), {recursive:true}); - let result = await activity.build(infile, callbacks, options); + let result = await activity.build(infile, outfile, callbacks, options); // check if we had a message that causes the build to be a failure // note: command line option here, if set, overrides project setting diff --git a/developer/src/kmc/src/commands/buildTestData/index.ts b/developer/src/kmc/src/commands/buildTestData/index.ts index 834112bc13..4253f55cf6 100644 --- a/developer/src/kmc/src/commands/buildTestData/index.ts +++ b/developer/src/kmc/src/commands/buildTestData/index.ts @@ -1,12 +1,13 @@ import * as fs from 'fs'; import * as path from 'path'; import * as kmcLdml from '@keymanapp/kmc-ldml'; -import { CompilerBaseOptions, CompilerCallbacks, defaultCompilerOptions, LDMLKeyboardTestDataXMLSourceFile, LDMLKeyboardXMLSourceFileReader } from '@keymanapp/common-types'; +import { CompilerCallbacks, defaultCompilerOptions, LDMLKeyboardTestDataXMLSourceFile, LDMLKeyboardXMLSourceFileReader } from '@keymanapp/common-types'; import { NodeCompilerCallbacks } from '../../util/NodeCompilerCallbacks.js'; import { fileURLToPath } from 'url'; +import { CommandLineBaseOptions } from 'src/util/baseOptions.js'; -export function buildTestData(infile: string, _options: any, commander: any) { - const options: CompilerBaseOptions = commander.optsWithGlobals(); +export async function buildTestData(infile: string, _options: any, commander: any) { + const options: CommandLineBaseOptions = commander.optsWithGlobals(); let compilerOptions: kmcLdml.LdmlCompilerOptions = { ...defaultCompilerOptions, @@ -18,7 +19,7 @@ export function buildTestData(infile: string, _options: any, commander: any) { } }; - let testData = loadTestData(infile, compilerOptions); + let testData = await loadTestData(infile, compilerOptions); if (!testData) { return; } @@ -31,12 +32,11 @@ export function buildTestData(infile: string, _options: any, commander: any) { fs.writeFileSync(outFileJson, JSON.stringify(testData, null, ' ')); } -function loadTestData(inputFilename: string, options: kmcLdml.LdmlCompilerOptions): LDMLKeyboardTestDataXMLSourceFile { +async function loadTestData(inputFilename: string, options: kmcLdml.LdmlCompilerOptions): Promise { const callbacks: CompilerCallbacks = new NodeCompilerCallbacks(options); - const k = new kmcLdml.LdmlKeyboardCompiler(callbacks, options); - let source = k.loadTestData(inputFilename); - if (!source) { + const k = new kmcLdml.LdmlKeyboardCompiler(); + if(!await k.init(callbacks, options)) { return null; } - return source; + return await k.loadTestData(inputFilename); } diff --git a/developer/src/kmc/src/commands/buildWindowsPackageInstaller/index.ts b/developer/src/kmc/src/commands/buildWindowsPackageInstaller/index.ts index 90bd30ce7f..5d64908856 100644 --- a/developer/src/kmc/src/commands/buildWindowsPackageInstaller/index.ts +++ b/developer/src/kmc/src/commands/buildWindowsPackageInstaller/index.ts @@ -1,10 +1,11 @@ import * as fs from 'fs'; import * as path from 'path'; -import { CompilerBaseOptions, CompilerCallbacks, defaultCompilerOptions } from '@keymanapp/common-types'; +import { CompilerCallbacks, defaultCompilerOptions } from '@keymanapp/common-types'; import { NodeCompilerCallbacks } from '../../util/NodeCompilerCallbacks.js'; import { WindowsPackageInstallerCompiler, WindowsPackageInstallerSources } from '@keymanapp/kmc-package'; +import { CommandLineBaseOptions } from 'src/util/baseOptions.js'; -interface WindowsPackageInstallerOptions extends CompilerBaseOptions { +interface WindowsPackageInstallerCommandLineOptions extends CommandLineBaseOptions { msi: string; exe: string; license: string; @@ -15,7 +16,9 @@ interface WindowsPackageInstallerOptions extends CompilerBaseOptions { }; export async function buildWindowsPackageInstaller(infile: string, _options: any, commander: any) { - const options: WindowsPackageInstallerOptions = commander.optsWithGlobals(); + // TODO(lowpri): we probably should cleanup the options management here, move + // translation of command line options to kmc-* options into a separate module + const options: WindowsPackageInstallerCommandLineOptions = commander.optsWithGlobals(); const sources: WindowsPackageInstallerSources = { licenseFilename: options.license, msiFilename: options.msi, @@ -26,22 +29,30 @@ export async function buildWindowsPackageInstaller(infile: string, _options: any titleImageFilename: options.titleImage } + const outfile: string = options.outFile; + // Normalize case for the filename and expand the path; this avoids false // positive case mismatches on input filenames and glommed paths infile = fs.realpathSync.native(infile); const callbacks: CompilerCallbacks = new NodeCompilerCallbacks({...defaultCompilerOptions, ...options}); - const compiler = new WindowsPackageInstallerCompiler(callbacks); + const compiler = new WindowsPackageInstallerCompiler(); + if(!await compiler.init(callbacks, {...options, sources})) { + process.exit(1); + } - const buffer = await compiler.compile(infile, sources); - if(!buffer) { + const fileBaseName = outfile ?? infile; + const outFileBase = path.basename(fileBaseName, path.extname(fileBaseName)); + const outFileDir = path.dirname(fileBaseName); + const outFileExe = path.join(outFileDir, outFileBase + '.exe'); + + const result = await compiler.run(infile, outFileExe); + if(!result) { // errors will have been reported already process.exit(1); } - const fileBaseName = options.outFile ?? infile; - const outFileBase = path.basename(fileBaseName, path.extname(fileBaseName)); - const outFileDir = path.dirname(fileBaseName); - const outFileExe = path.join(outFileDir, outFileBase + '.exe'); - fs.writeFileSync(outFileExe, buffer); + if(!await compiler.write(result.artifacts)) { + process.exit(1); + } } diff --git a/developer/src/kmc/src/kmlmc.ts b/developer/src/kmc/src/kmlmc.ts index 1ce171fd15..ca2fd4d6b0 100644 --- a/developer/src/kmc/src/kmlmc.ts +++ b/developer/src/kmc/src/kmlmc.ts @@ -3,9 +3,8 @@ * kmlmc - Keyman Lexical Model Compiler */ -import * as fs from 'fs'; import { Command } from 'commander'; -import { compileModel } from '@keymanapp/kmc-model'; +import { LexicalModelCompiler } from '@keymanapp/kmc-model'; import { SysExits } from './util/sysexits.js'; import KEYMAN_VERSION from "@keymanapp/keyman-version"; import { NodeCompilerCallbacks } from './util/NodeCompilerCallbacks.js'; @@ -30,10 +29,16 @@ if (!inputFilename) { const callbacks = new NodeCompilerCallbacks({logLevel: 'info'}); +const compiler = new LexicalModelCompiler(); +if(!await compiler.init(callbacks, null)) { + console.error('Initialization failed.'); + process.exit(SysExits.EX_DATAERR); +} + let code = null; // Compile: try { - code = compileModel(inputFilename, callbacks); + code = await compiler.run(inputFilename, program.opts().outFile); } catch(e) { console.error(e); process.exit(SysExits.EX_DATAERR); @@ -46,9 +51,12 @@ if(!code) { // Output: if (program.opts().outFile) { - fs.writeFileSync(program.opts().outFile, code, 'utf8'); + compiler.write(code.artifacts); } else { - console.log(code); + // TODO(lowpri): if writing to console then log messages should all be to stderr? + const decoder = new TextDecoder(); + const text = decoder.decode(code.artifacts.js.data); + console.log(text); } function exitDueToUsageError(message: string): never { diff --git a/developer/src/kmc/src/kmlmp.ts b/developer/src/kmc/src/kmlmp.ts index ae629f073e..bb0cda5e53 100644 --- a/developer/src/kmc/src/kmlmp.ts +++ b/developer/src/kmc/src/kmlmp.ts @@ -5,9 +5,8 @@ // Note: this is a deprecated package and will be removed in Keyman 18.0 -import * as fs from 'fs'; import { Command } from 'commander'; -import { PackageValidation, KmpCompiler } from '@keymanapp/kmc-package'; +import { KmpCompiler } from '@keymanapp/kmc-package'; import { SysExits } from './util/sysexits.js'; import KEYMAN_VERSION from "@keymanapp/keyman-version"; import { NodeCompilerCallbacks } from './util/NodeCompilerCallbacks.js'; @@ -34,36 +33,24 @@ if (!inputFilename) { let outputFilename: string = program.opts().outFile ? program.opts().outFile : inputFilename.replace(/\.kps$/, ".kmp"); // -// Load .kps source data +// Run the compiler // const callbacks = new NodeCompilerCallbacks({logLevel: 'info'}); -let kmpCompiler = new KmpCompiler(callbacks); -let kmpJsonData = kmpCompiler.transformKpsToKmpObject(inputFilename); -if(!kmpJsonData) { +let kmpCompiler = new KmpCompiler(); +if(!await kmpCompiler.init(callbacks, null)) { process.exit(1); } -// -// Validate the package file -// - -const validation = new PackageValidation(callbacks, {}); -if(!validation.validate(inputFilename, kmpJsonData)) { +let result = await kmpCompiler.run(inputFilename, outputFilename); +if(!result) { process.exit(1); } -// -// Build the .kmp package file -// - -let promise = kmpCompiler.buildKmpFile(inputFilename, kmpJsonData); -promise.then(data => { - fs.writeFileSync(outputFilename, data, 'binary'); -}).catch(error => { - // Consumer decides how to handle errors +if(!await kmpCompiler.write(result.artifacts)) { console.error('Failed to write kmp file'); -}); + process.exit(1); +} function exitDueToUsageError(message: string): never { console.error(`${program.name()}: ${message}`); diff --git a/developer/src/kmc/src/messages/infrastructureMessages.ts b/developer/src/kmc/src/messages/infrastructureMessages.ts index 18c88cf5c3..0100bdccf7 100644 --- a/developer/src/kmc/src/messages/infrastructureMessages.ts +++ b/developer/src/kmc/src/messages/infrastructureMessages.ts @@ -91,5 +91,9 @@ export class InfrastructureMessages { static Hint_ProjectIsVersion10 = () => m(this.HINT_ProjectIsVersion10, `The project file is an older version and can be upgraded to version 17.0`); static HINT_ProjectIsVersion10 = SevHint | 0x0014; + + static Error_OutFileCanOnlyBeSpecifiedWithSingleInfile = () => m(this.ERROR_OutFileCanOnlyBeSpecifiedWithSingleInfile, + `Parameter --out-file can only be used with a single input file.`); + static ERROR_OutFileCanOnlyBeSpecifiedWithSingleInfile = SevError | 0x0015; } diff --git a/developer/src/kmc/src/util/TestKeymanSentry.ts b/developer/src/kmc/src/util/TestKeymanSentry.ts index 75da80e53d..b1c4aa40ee 100644 --- a/developer/src/kmc/src/util/TestKeymanSentry.ts +++ b/developer/src/kmc/src/util/TestKeymanSentry.ts @@ -26,7 +26,7 @@ export class TestKeymanSentry { if(cli.includes('kmcmplib')) { const compiler = new KmnCompiler(); const callbacks = new NodeCompilerCallbacks({}); - if(!await compiler.init(callbacks)) { + if(!await compiler.init(callbacks, null)) { throw new Error('Failed to instantiate WASM compiler'); } try { diff --git a/developer/src/kmc/src/util/baseOptions.ts b/developer/src/kmc/src/util/baseOptions.ts index 9416376951..dd91f0bae6 100644 --- a/developer/src/kmc/src/util/baseOptions.ts +++ b/developer/src/kmc/src/util/baseOptions.ts @@ -1,8 +1,23 @@ -import { ALL_COMPILER_LOG_FORMATS, ALL_COMPILER_LOG_LEVELS } from "@keymanapp/common-types"; +import { ALL_COMPILER_LOG_FORMATS, ALL_COMPILER_LOG_LEVELS, CompilerLogFormat, CompilerLogLevel } from "@keymanapp/common-types"; import { Command, Option } from "commander"; -// These options map to CompilerBaseOptions +/** + * Abstract interface for compiler options + */ +export interface CommandLineBaseOptions { + // These options map to CompilerBaseOptions + logLevel?: CompilerLogLevel; + logFormat?: CompilerLogFormat; + color?: boolean; + + // This option is not in CompilerBaseOptions + outFile?:string; +} + +/** + * These options map to CompilerBaseOptions + */ export class BaseOptions { public static addLogLevel(program: Command) { return program.addOption(new Option('-l, --log-level ', 'Log level').choices(ALL_COMPILER_LOG_LEVELS).default('info')); diff --git a/developer/src/kmc/src/util/escapeRegExp.ts b/developer/src/kmc/src/util/escapeRegExp.ts deleted file mode 100644 index 8972b6bfed..0000000000 --- a/developer/src/kmc/src/util/escapeRegExp.ts +++ /dev/null @@ -1,7 +0,0 @@ - -const escapedRegexp = /[.*+?^${}()|[\]\\]/g; - -// See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions#escaping -export function escapeRegExp(s: string) { - return s.replace(escapedRegexp, "\\$&"); // $& means the whole matched string -} diff --git a/developer/src/kmc/test/test-build.ts b/developer/src/kmc/test/test-build.ts index 26ef09e746..d859e3f26d 100644 --- a/developer/src/kmc/test/test-build.ts +++ b/developer/src/kmc/test/test-build.ts @@ -34,7 +34,7 @@ describe('compilerWarningsAsErrors', function () { const builder = new BuildProject(); const path = makePathToFixture('compiler-warnings-as-errors', `compiler_warnings_as_errors_${truth.kpj === true ? 'true' : (truth.kpj === false ? 'false' : 'undefined')}.kpj`); - const result = await builder.build(path, callbacks, { + const result = await builder.build(path, null, callbacks, { compilerWarningsAsErrors: truth.cli, }); if(truth.result != result) { diff --git a/developer/src/kmc/test/test-infrastructureMessages.ts b/developer/src/kmc/test/test-infrastructureMessages.ts index 6d543a5fd7..34898a5ad9 100644 --- a/developer/src/kmc/test/test-infrastructureMessages.ts +++ b/developer/src/kmc/test/test-infrastructureMessages.ts @@ -24,7 +24,7 @@ describe('InfrastructureMessages', function () { const expectedMessages = [InfrastructureMessages.FATAL_UnexpectedException]; process.env.SENTRY_CLIENT_TEST_BUILD_EXCEPTION = '1'; - await unitTestEndpoints.build(null, ncb, {}); + await unitTestEndpoints.build(null, null, ncb, {}); delete process.env.SENTRY_CLIENT_TEST_BUILD_EXCEPTION; assertMessagesEqual(ncb.messages, expectedMessages); @@ -79,7 +79,7 @@ describe('InfrastructureMessages', function () { InfrastructureMessages.INFO_WarningsHaveFailedBuild, InfrastructureMessages.INFO_FileNotBuiltSuccessfully ]; - await unitTestEndpoints.build(filename, ncb, {compilerWarningsAsErrors: true}); + await unitTestEndpoints.build(filename, null, ncb, {compilerWarningsAsErrors: true}); assertMessagesEqual(ncb.messages, expectedMessages); }); @@ -93,7 +93,7 @@ describe('InfrastructureMessages', function () { InfrastructureMessages.ERROR_UnsupportedProjectVersion, InfrastructureMessages.INFO_ProjectNotBuiltSuccessfully ]; - await unitTestEndpoints.build(filename, ncb, {compilerWarningsAsErrors: true}); + await unitTestEndpoints.build(filename, null, ncb, {compilerWarningsAsErrors: true}); assertMessagesEqual(ncb.messages, expectedMessages); }); }); diff --git a/developer/src/kmc/test/test-project-build.ts b/developer/src/kmc/test/test-project-build.ts index c7e1c82bf5..83fd37a55f 100644 --- a/developer/src/kmc/test/test-project-build.ts +++ b/developer/src/kmc/test/test-project-build.ts @@ -11,7 +11,7 @@ describe('BuildProject', function () { it('should build a keyboard project', async function() { const builder = new BuildProject(); const path = makePathToFixture('relative_paths', 'k_000___null_keyboard.kpj'); - let result = await builder.build(path, callbacks, { + let result = await builder.build(path, null, callbacks, { shouldAddCompilerVersion: false, compilerWarningsAsErrors: true, saveDebug: false,