erabo/spiegel-keyman
1
0
Fork 0
mirror of https://github.com/keymanapp/keyman.git synced 2026-08-13 04:09:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 22

Merge pull request #10208 from keymanapp/feat/developer/9473-kmc-module-api-consolidation

Browse source 
feat(developer): Consolidate public APIs for kmc modules
This commit is contained in:
Marc Durdin 2024-01-02 15:35:47 +11:00 • committed by GitHub
commit edc950727f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
58 changed files with 919 additions and 653 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

62
common/web/input-processor/tests/cases/languageProcessor.js
View file

@ -57,7 +57,13 @@ describe('LanguageProcessor', function() {
});
describe('.predict', function() {
let compiler = new LexicalModelCompiler(callbacks);
let compiler = null;
this.beforeAll(async function() {
compiler = new LexicalModelCompiler();
assert.isTrue(await compiler.init(callbacks, {}));
});
const MODEL_ID = 'example.qaa.trivial';
// ES-module mode leaves out `__dirname`, so we rebuild it using other components.
@ -67,20 +73,23 @@ describe('LanguageProcessor', function() {
const PATH = path.join(__dirname, '../../../../../developer/src/kmc-model/test/fixtures', MODEL_ID);
describe('using angle brackets for quotes', function() {
let modelCode = compiler.generateLexicalModelCode(MODEL_ID, {
format: 'trie-1.0',
sources: ['wordlist.tsv'],
punctuation: {
quotesForKeepSuggestion: { open: `«`, close: `»`},
insertAfterWord: " " , // OGHAM SPACE MARK
}
}, PATH);
let modelCode = null, modelSpec = null;
this.beforeAll(function() {
modelCode = compiler.generateLexicalModelCode(MODEL_ID, {
format: 'trie-1.0',
sources: ['wordlist.tsv'],
punctuation: {
quotesForKeepSuggestion: { open: `«`, close: `»`},
insertAfterWord: " " , // OGHAM SPACE MARK
}
}, PATH);
let modelSpec = {
id: MODEL_ID,
languages: ['en'],
code: modelCode
};
modelSpec = {
id: MODEL_ID,
languages: ['en'],
code: modelCode
};
});
it("successfully loads the model", function(done) {
let languageProcessor = new LanguageProcessor(worker, new TranscriptionCache());
@ -115,18 +124,21 @@ describe('LanguageProcessor', function() {
});
describe('properly cases generated suggestions', function() {
let modelCode = compiler.generateLexicalModelCode(MODEL_ID, {
format: 'trie-1.0',
sources: ['wordlist.tsv'],
languageUsesCasing: true,
//applyCasing // we rely on the compiler's default implementation here.
}, PATH);
let modelCode = null, modelSpec = null;
this.beforeAll(function () {
modelCode = compiler.generateLexicalModelCode(MODEL_ID, {
format: 'trie-1.0',
sources: ['wordlist.tsv'],
languageUsesCasing: true,
//applyCasing // we rely on the compiler's default implementation here.
}, PATH);
let modelSpec = {
id: MODEL_ID,
languages: ['en'],
code: modelCode
};
modelSpec = {
id: MODEL_ID,
languages: ['en'],
code: modelCode
};
});
describe("does not alter casing when input is lowercased", function() {
it("when input is fully lowercased", function(done) {

12
common/web/types/src/kpj/keyman-developer-project.ts
View file

@ -37,8 +37,16 @@ export class KeymanDeveloperProject {
for(let filename of files) {
let fullPath = this.callbacks.path.join(sourcePath, filename);
if(KeymanFileTypes.filenameIs(filename, KeymanFileTypes.Source.LdmlKeyboard)) {
if(!this.callbacks.fs.readFileSync(fullPath, 'utf-8').match(/ldmlKeyboard3\.dtd/)) {
// Skip this .xml because we assume it isn't really a keyboard .xml
try {
const data = this.callbacks.loadFile(fullPath);
const text = new TextDecoder().decode(data);
if(!text?.match(/ldmlKeyboard3\.dtd/)) {
// Skip this .xml because we assume it isn't really a keyboard .xml
continue;
}
} catch(e) {
// We'll just silently skip this file because we were not able to load it,
// so let's hope it wasn't a real LDML keyboard XML :-)
continue;
}
}

7
common/web/types/src/main.ts
View file

@ -27,6 +27,13 @@ export { defaultCompilerOptions, CompilerBaseOptions, CompilerCallbacks, Compile
compilerExceptionToString, compilerErrorFormatCode,
compilerLogLevelToSeverity, CompilerLogLevel, compilerEventFormat, ALL_COMPILER_LOG_LEVELS,
ALL_COMPILER_LOG_FORMATS, CompilerLogFormat,
KeymanCompilerArtifact,
KeymanCompilerArtifactOptional,
KeymanCompilerArtifacts,
KeymanCompilerResult,
KeymanCompiler
} from './util/compiler-interfaces.js';
export { CommonTypesMessages } from './util/common-events.js';

38
common/web/types/src/util/compiler-interfaces.ts
View file

@ -253,6 +253,40 @@ export interface CompilerCallbackOptions {
compilerWarningsAsErrors?: boolean;
};
export interface KeymanCompilerArtifact {
data: Uint8Array;
filename: string;
};
export type KeymanCompilerArtifactOptional = KeymanCompilerArtifact | undefined;
export interface KeymanCompilerArtifacts {
readonly [type:string]: KeymanCompilerArtifactOptional;
};
export interface KeymanCompilerResult {
artifacts: KeymanCompilerArtifacts;
};
export interface KeymanCompiler {
init(callbacks: CompilerCallbacks, options: CompilerOptions): Promise<boolean>;
/**
* Run the compiler, and save the result in memory arrays. Note that while
* `outputFilename` is provided here, the output file is not written to in
* this function.
* @param inputFilename
* @param outputFilename The intended output filename, optional, if missing,
* calculated from inputFilename
* @param data
*/
run(inputFilename:string, outputFilename?:string /*, data?: any*/): Promise<KeymanCompilerResult>;
/**
* Writes the compiled output files to disk
* @param artifacts
*/
write(artifacts: KeymanCompilerArtifacts): Promise<boolean>;
};
/**
* Abstract interface for callbacks, to abstract out file i/o
*/
@ -369,10 +403,6 @@ export interface CompilerBaseOptions {
* Format of output for log to console
*/
logFormat?: CompilerLogFormat;
/**
* Optional output file for activities that generate output
*/
outFile?: string;
/**
* Colorize log output, default is detected from console
*/

12
developer/src/kmc-analyze/src/util/get-osk-from-kmn-file.ts
View file

@ -9,15 +9,15 @@ export async function getOskFromKmnFile(callbacks: CompilerCallbacks, filename:
let touchLayoutFilename: string;
const kmnCompiler = new KmnCompiler();
if(!await kmnCompiler.init(callbacks)) {
if(!await kmnCompiler.init(callbacks, {
shouldAddCompilerVersion: false,
saveDebug: false,
})) {
// kmnCompiler will report errors
return null;
}
let result = kmnCompiler.runCompiler(filename, {
shouldAddCompilerVersion: false,
saveDebug: false,
});
let result = await kmnCompiler.run(filename, null);
if(!result) {
// kmnCompiler will report any errors
@ -29,7 +29,7 @@ export async function getOskFromKmnFile(callbacks: CompilerCallbacks, filename:
}
const reader = new KmxFileReader();
const keyboard: KMX.KEYBOARD = reader.read(result.kmx.data);
const keyboard: KMX.KEYBOARD = reader.read(result.artifacts.kmx.data);
const touchLayoutStore = keyboard.stores.find(store => store.dwSystemID == KMX.KMXFile.TSS_LAYOUTFILE);
if(touchLayoutStore) {

59
developer/src/kmc-keyboard-info/src/index.ts
View file

@ -5,7 +5,7 @@
import { minKeymanVersion } from "./min-keyman-version.js";
import { KeyboardInfoFile, KeyboardInfoFileIncludes, KeyboardInfoFileLanguageFont, KeyboardInfoFilePlatform } from "./keyboard-info-file.js";
import { KeymanFileTypes, CompilerCallbacks, KmpJsonFile, KmxFileReader, KMX, KeymanTargets } from "@keymanapp/common-types";
import { KeymanFileTypes, CompilerCallbacks, KmpJsonFile, KmxFileReader, KMX, KeymanTargets, KeymanCompiler, CompilerOptions, KeymanCompilerResult, KeymanCompilerArtifacts, KeymanCompilerArtifact } from "@keymanapp/common-types";
import { KeyboardInfoCompilerMessages } from "./messages.js";
import langtags from "./imports/langtags.js";
import { validateMITLicense } from "@keymanapp/developer-utils";
@ -24,7 +24,7 @@ const HelpRoot = 'https://help.keyman.com/keyboard/';
* Build a dictionary of language tags from langtags.json
*/
function init(): void {
function preinit(): void {
if(langtagsByTag['en']) {
// Already initialized, we can reasonably assume that 'en' will always be in
// langtags.json.
@ -62,9 +62,30 @@ export interface KeyboardInfoSources {
forPublishing: boolean;
};
export class KeyboardInfoCompiler {
constructor(private callbacks: CompilerCallbacks) {
init();
export interface KeyboardInfoCompilerOptions extends CompilerOptions {
sources: KeyboardInfoSources;
};
export interface KeyboardInfoCompilerArtifacts extends KeymanCompilerArtifacts {
keyboard_info: KeymanCompilerArtifact;
};
export interface KeyboardInfoCompilerResult extends KeymanCompilerResult {
artifacts: KeyboardInfoCompilerArtifacts;
};
export class KeyboardInfoCompiler implements KeymanCompiler {
private callbacks: CompilerCallbacks;
private options: KeyboardInfoCompilerOptions;
constructor() {
preinit();
}
public async init(callbacks: CompilerCallbacks, options: KeyboardInfoCompilerOptions): Promise<boolean> {
this.callbacks = callbacks;
this.options = {...options};
return true;
}
/**
@ -77,15 +98,18 @@ export class KeyboardInfoCompiler {
*
* @param sources Details on files from which to extract metadata
*/
public async writeKeyboardInfoFile(
sources: KeyboardInfoSources
): Promise<Uint8Array> {
public async run(inputFilename: string, outputFilename?: string): Promise<KeyboardInfoCompilerResult> {
const sources = this.options.sources;
// TODO(lowpri): work from .kpj and nothing else as input. Blocked because
// .kpj work is largely in kmc at present, so that would need to move to
// a separate module.
const kmpCompiler = new KmpCompiler(this.callbacks);
const kmpCompiler = new KmpCompiler();
if(!await kmpCompiler.init(this.callbacks, {})) {
// Errors will have been emitted by KmpCompiler
return null;
}
const kmpJsonData = kmpCompiler.transformKpsToKmpObject(sources.kpsFilename);
if(!kmpJsonData) {
// Errors will have been emitted by KmpCompiler
@ -307,7 +331,22 @@ export class KeyboardInfoCompiler {
}, null, 2));
}
return new TextEncoder().encode(jsonOutput);
const data = new TextEncoder().encode(jsonOutput);
const result: KeyboardInfoCompilerResult = {
artifacts: {
keyboard_info: {
data,
filename: outputFilename ?? inputFilename.replace(/\.kpj$/, '.keyboard_info')
}
}
};
return result;
}
public async write(artifacts: KeyboardInfoCompilerArtifacts): Promise<boolean> {
this.callbacks.fs.writeFileSync(artifacts.keyboard_info.filename, artifacts.keyboard_info.data);
return true;
}
private mapKeymanTargetToPlatform(target: KeymanTargets.KeymanTarget): KeyboardInfoFilePlatform[] {

30
developer/src/kmc-keyboard-info/test/test-keyboard-info-compiler.ts
View file

@ -3,7 +3,7 @@ import { assert } from 'chai';
import 'mocha';
import { TestCompilerCallbacks } from '@keymanapp/developer-test-helpers';
import { makePathToFixture } from './helpers/index.js';
import { KeyboardInfoCompiler } from '../src/index.js';
import { KeyboardInfoCompiler, KeyboardInfoCompilerResult } from '../src/index.js';
const callbacks = new TestCompilerCallbacks();
@ -13,31 +13,35 @@ beforeEach(function() {
describe('keyboard-info-compiler', function () {
it('compile a .keyboard_info file correctly', async function() {
const kpjFilename = makePathToFixture('khmer_angkor', 'khmer_angkor.kpj');
const jsFilename = makePathToFixture('khmer_angkor', 'build', 'khmer_angkor.js');
const kpsFilename = makePathToFixture('khmer_angkor', 'source', 'khmer_angkor.kps');
const kmpFilename = makePathToFixture('khmer_angkor', 'build', 'khmer_angkor.kmp');
const buildKeyboardInfoFilename = makePathToFixture('khmer_angkor', 'build', 'khmer_angkor.keyboard_info');
const compiler = new KeyboardInfoCompiler(callbacks);
let data = null;
const sources = {
kmpFilename,
sourcePath: 'release/k/khmer_angkor',
kpsFilename,
jsFilename: jsFilename,
forPublishing: true,
};
const compiler = new KeyboardInfoCompiler();
assert.isTrue(await compiler.init(callbacks, {sources}));
let result: KeyboardInfoCompilerResult = null;
try {
data = await compiler.writeKeyboardInfoFile({
kmpFilename,
sourcePath: 'release/k/khmer_angkor',
kpsFilename,
jsFilename: jsFilename,
forPublishing: true,
});
result = await compiler.run(kpjFilename, null);
} catch(e) {
callbacks.printMessages();
throw e;
}
if(data == null) {
if(result == null) {
callbacks.printMessages();
}
assert.isNotNull(data);
assert.isNotNull(result);
const actual = JSON.parse(new TextDecoder().decode(data));
const actual = JSON.parse(new TextDecoder().decode(result.artifacts.keyboard_info.data));
const expected = JSON.parse(fs.readFileSync(buildKeyboardInfoFilename, 'utf-8'));
// `lastModifiedDate` is dependent on time of run (not worth mocking)

96
developer/src/kmc-kmn/src/compiler/compiler.ts
View file

@ -6,17 +6,12 @@ TODO: implement additional interfaces:
*/
// TODO: rename wasm-host?
import { UnicodeSetParser, UnicodeSet, Osk, VisualKeyboard, KvkFileReader } from '@keymanapp/common-types';
import { UnicodeSetParser, UnicodeSet, Osk, VisualKeyboard, KvkFileReader, KeymanCompiler, KeymanCompilerArtifacts, KeymanCompilerArtifactOptional, KeymanCompilerResult, KeymanCompilerArtifact } from '@keymanapp/common-types';
import { CompilerCallbacks, CompilerEvent, CompilerOptions, KeymanFileTypes, KvkFileWriter, KvksFileReader } from '@keymanapp/common-types';
import loadWasmHost from '../import/kmcmplib/wasm-host.js';
import { CompilerMessages, mapErrorFromKmcmplib } from './kmn-compiler-messages.js';
import { WriteCompiledKeyboard } from '../kmw-compiler/kmw-compiler.js';
export interface CompilerResultFile {
filename: string;
data: Uint8Array;
};
//
// Matches kmcmplibapi.h definitions
//
@ -46,7 +41,7 @@ export const COMPILETARGETS__MASK = 0x03;
/**
* Data in CompilerResultExtra comes from kmcmplib
*/
export interface CompilerResultExtra {
export interface KmnCompilerResultExtra {
/**
* A bitmask, consisting of COMPILETARGETS_KMX and/or COMPILETARGETS_JS
*/
@ -61,11 +56,15 @@ export interface CompilerResultExtra {
// Internal in-memory result from a successful compilation
//
export interface CompilerResult {
kmx?: CompilerResultFile;
kvk?: CompilerResultFile;
js?: CompilerResultFile;
extra: CompilerResultExtra;
export interface KmnCompilerArtifacts extends KeymanCompilerArtifacts {
kmx?: KeymanCompilerArtifactOptional;
kvk?: KeymanCompilerArtifactOptional;
js?: KeymanCompilerArtifactOptional;
};
export interface KmnCompilerResult extends KeymanCompilerResult {
artifacts: KmnCompilerArtifacts;
extra: KmnCompilerResultExtra;
displayMap?: Osk.PuaMap;
};
@ -97,18 +96,20 @@ interface MallocAndFree {
let
Module: any;
export class KmnCompiler implements UnicodeSetParser {
callbackID: string; // a unique numeric id added to globals with prefixed names
callbacks: CompilerCallbacks;
wasmExports: MallocAndFree;
export class KmnCompiler implements KeymanCompiler, UnicodeSetParser {
private readonly callbackID: string; // a unique numeric id added to globals with prefixed names
private callbacks: CompilerCallbacks;
private wasmExports: MallocAndFree;
private options: KmnCompilerOptions;
constructor() {
this.callbackID = callbackPrefix + callbackProcIdentifier.toString();
callbackProcIdentifier++;
}
public async init(callbacks: CompilerCallbacks): Promise<boolean> {
public async init(callbacks: CompilerCallbacks, options: KmnCompilerOptions): Promise<boolean> {
this.callbacks = callbacks;
this.options = {...options};
if(!Module) {
try {
Module = await loadWasmHost();
@ -140,20 +141,22 @@ export class KmnCompiler implements UnicodeSetParser {
return true;
}
public run(infile: string, options?: KmnCompilerOptions): boolean {
let result = this.runCompiler(infile, options);
if(result) {
if(result.kmx) {
this.callbacks.fs.writeFileSync(result.kmx.filename, result.kmx.data);
}
if(result.kvk) {
this.callbacks.fs.writeFileSync(result.kvk.filename, result.kvk.data);
}
if(result.js) {
this.callbacks.fs.writeFileSync(result.js.filename, result.js.data);
}
public async write(artifacts: KmnCompilerArtifacts): Promise<boolean> {
if(!artifacts) {
throw Error('artifacts must be defined');
}
return !!result;
if(artifacts.kmx) {
this.callbacks.fs.writeFileSync(artifacts.kmx.filename, artifacts.kmx.data);
}
if(artifacts.kvk) {
this.callbacks.fs.writeFileSync(artifacts.kvk.filename, artifacts.kvk.data);
}
if(artifacts.js) {
this.callbacks.fs.writeFileSync(artifacts.js.filename, artifacts.js.data);
}
return true;
}
private compilerMessageCallback = (line: number, code: number, msg: string): number => {
@ -196,9 +199,10 @@ export class KmnCompiler implements UnicodeSetParser {
return 1;
}
private copyWasmResult(wasm_result: any): CompilerResult {
let result: CompilerResult = {
private copyWasmResult(wasm_result: any): KmnCompilerResult {
let result: KmnCompilerResult = {
// We cannot Object.assign or {...} on a wasm-defined object, so...
artifacts: {},
extra: {
targets: wasm_result.extra.targets,
displayMapFilename: wasm_result.extra.displayMapFilename,
@ -234,15 +238,15 @@ export class KmnCompiler implements UnicodeSetParser {
return new Uint8Array(new Uint8Array(Module.HEAP8.buffer, offset, size));
}
public runCompiler(infile: string, options: KmnCompilerOptions): CompilerResult {
public async run(infile: string, outfile: string): Promise<KmnCompilerResult> {
if(!this.verifyInitialized()) {
/* c8 ignore next 2 */
return null;
}
options = {...baseOptions, ...options};
const options = {...baseOptions, ...this.options};
options.outFile = options.outFile ?? infile.replace(/\.kmn$/i, '.kmx');
outfile = outfile ?? infile.replace(/\.kmn$/i, '.kmx');
(globalThis as any)[this.callbackID] = {
message: this.compilerMessageCallback,
@ -264,11 +268,11 @@ export class KmnCompiler implements UnicodeSetParser {
return null;
}
const result: CompilerResult = this.copyWasmResult(wasm_result);
const result: KmnCompilerResult = this.copyWasmResult(wasm_result);
if(result.extra.targets & COMPILETARGETS_KMX) {
result.kmx = {
filename: options.outFile,
result.artifacts.kmx = {
filename: outfile,
data: this.copyWasmBuffer(wasm_result.kmx, wasm_result.kmxSize)
};
}
@ -286,8 +290,8 @@ export class KmnCompiler implements UnicodeSetParser {
}
if(result.extra.kvksFilename) {
result.kvk = this.runKvkCompiler(result.extra.kvksFilename, infile, options.outFile, result.displayMap);
if(!result.kvk) {
result.artifacts.kvk = this.runKvkCompiler(result.extra.kvksFilename, infile, outfile, result.displayMap);
if(!result.artifacts.kvk) {
return null;
}
}
@ -308,12 +312,12 @@ export class KmnCompiler implements UnicodeSetParser {
if(!wasm_result.result) {
return null;
}
const kmw_result: CompilerResult = this.copyWasmResult(wasm_result);
const kmw_result: KmnCompilerResult = this.copyWasmResult(wasm_result);
kmw_result.displayMap = result.displayMap; // we can safely re-use the kmx compile displayMap
const web_kmx = this.copyWasmBuffer(wasm_result.kmx, wasm_result.kmxSize);
result.js = this.runWebCompiler(infile, options.outFile, web_kmx, result.kvk?.data, kmw_result, options);
if(!result.js) {
result.artifacts.js = this.runWebCompiler(infile, outfile, web_kmx, result.artifacts.kvk?.data, kmw_result, options);
if(!result.artifacts.js) {
return null;
}
}
@ -338,9 +342,9 @@ export class KmnCompiler implements UnicodeSetParser {
kmxFilename: string,
web_kmx: Uint8Array,
kvk: Uint8Array,
kmxResult: CompilerResult,
kmxResult: KmnCompilerResult,
options: CompilerOptions
): CompilerResultFile {
): KeymanCompilerArtifact {
const data = WriteCompiledKeyboard(this.callbacks, kmnFilename, web_kmx, kvk, kmxResult, options.saveDebug);
if(!data) {
return null;
@ -348,7 +352,7 @@ export class KmnCompiler implements UnicodeSetParser {
return {
filename: this.callbacks.path.join(this.callbacks.path.dirname(kmxFilename),
this.keyboardIdFromKmnFilename(kmnFilename) + KeymanFileTypes.Binary.WebKeyboard),
this.keyboardIdFromKmnFilename(kmnFilename) + KeymanFileTypes.Binary.WebKeyboard),
data: new TextEncoder().encode(data)
};
}

6
developer/src/kmc-kmn/src/kmw-compiler/compiler-globals.ts
View file

@ -1,10 +1,10 @@
import { KMX, CompilerCallbacks, CompilerOptions } from "@keymanapp/common-types";
import { CompilerResult } from "../compiler/compiler.js";
import { KmnCompilerResult } from "../compiler/compiler.js";
export let FTabStop: string;
export let nl: string;
export let FCompilerWarningsAsErrors = false;
export let kmxResult: CompilerResult;
export let kmxResult: KmnCompilerResult;
export let fk: KMX.KEYBOARD;
export let FMnemonic: boolean;
export let options: CompilerOptions;
@ -19,7 +19,7 @@ export function setupGlobals(
_options: CompilerOptions,
_tab: string,
_nl: string,
_kmxResult: CompilerResult,
_kmxResult: KmnCompilerResult,
_keyboard: KMX.KEYBOARD,
_kmnfile: string
) {

4
developer/src/kmc-kmn/src/kmw-compiler/kmw-compiler.ts
View file

@ -5,7 +5,7 @@ import { JavaScript_ContextMatch, JavaScript_KeyAsString, JavaScript_Name, JavaS
import { KmwCompilerMessages } from "./kmw-compiler-messages.js";
import { ValidateLayoutFile } from "./validate-layout-file.js";
import { VisualKeyboardFromFile } from "./visual-keyboard-compiler.js";
import { CompilerResult, STORETYPE_DEBUG, STORETYPE_OPTION, STORETYPE_RESERVED } from "../compiler/compiler.js";
import { KmnCompilerResult, STORETYPE_DEBUG, STORETYPE_OPTION, STORETYPE_RESERVED } from "../compiler/compiler.js";
function requote(s: string): string {
return "'" + s.replaceAll(/(['\\])/g, "\\$1") + "'";
@ -41,7 +41,7 @@ export function WriteCompiledKeyboard(
kmnfile: string,
keyboardData: Uint8Array,
kvkData: Uint8Array,
kmxResult: CompilerResult,
kmxResult: KmnCompilerResult,
FDebug: boolean = false
): string {
let opts: CompilerOptions = {

17
developer/src/kmc-kmn/test/kmw/test-kmw-compiler.ts
View file

@ -5,7 +5,7 @@ import { dirname } from 'path';
import { fileURLToPath } from 'url';
import fs from 'fs';
import { TestCompilerCallbacks } from '@keymanapp/developer-test-helpers';
import { CompilerResult, KmnCompiler } from '../../src/compiler/compiler.js';
import { KmnCompilerResult, KmnCompiler } from '../../src/compiler/compiler.js';
import { ETLResult, extractTouchLayout as parseWebTestResult } from './util.js';
import { KeymanFileTypes } from '@keymanapp/common-types';
@ -27,7 +27,10 @@ describe('KeymanWeb Compiler', function() {
const kmnCompiler: KmnCompiler = new KmnCompiler();
this.beforeAll(async function() {
assert.isTrue(await kmnCompiler.init(callbacks));
assert.isTrue(await kmnCompiler.init(callbacks, {
shouldAddCompilerVersion: false,
saveDebug: true,
}));
});
this.afterEach(function() {
@ -79,18 +82,16 @@ describe('KeymanWeb Compiler', function() {
});
function run_test_keyboard(kmnCompiler: KmnCompiler, id: string): { result: CompilerResult, actualCode: string, actual: ETLResult, expectedCode: string, expected: ETLResult } {
async function run_test_keyboard(kmnCompiler: KmnCompiler, id: string):
Promise<{ result: KmnCompilerResult, actualCode: string, actual: ETLResult, expectedCode: string, expected: ETLResult }> {
const filenames = generateTestFilenames(id);
let result = kmnCompiler.runCompiler(filenames.source, {
shouldAddCompilerVersion: false,
saveDebug: true,
});
let result = await kmnCompiler.run(filenames.source, null);
assert.isNotNull(result);
let value = {
result,
actualCode: new TextDecoder().decode(result.js.data),
actualCode: new TextDecoder().decode(result.artifacts.js.data),
expectedCode: fs.readFileSync(filenames.fixture, 'utf8'),
expected: <ETLResult>null,
actual: <ETLResult>null,

45
developer/src/kmc-kmn/test/test-compiler.ts
View file

@ -15,7 +15,7 @@ describe('Compiler class', function() {
const compiler = new KmnCompiler();
const callbacks : any = null; // ERROR
try {
await compiler.init(callbacks)
await compiler.init(callbacks, null)
assert.fail('Expected exception');
} catch(e) {
assert.ok(e);
@ -26,21 +26,27 @@ describe('Compiler class', function() {
it('should start', async function() {
const compiler = new KmnCompiler();
const callbacks = new TestCompilerCallbacks();
assert(await compiler.init(callbacks));
assert(await compiler.init(callbacks, null));
assert(compiler.verifyInitialized());
});
it('should compile a basic keyboard', async function() {
const compiler = new KmnCompiler();
const callbacks = new TestCompilerCallbacks();
assert(await compiler.init(callbacks));
assert(await compiler.init(callbacks, {saveDebug: true, shouldAddCompilerVersion: false}));
assert(compiler.verifyInitialized());
const fixtureName = baselineDir + 'k_000___null_keyboard.kmx';
const infile = baselineDir + 'k_000___null_keyboard.kmn';
const outFile = __dirname + '/k_000___null_keyboard.kmx';
assert(compiler.run(infile, {saveDebug: true, outFile, shouldAddCompilerVersion: false}));
if(fs.existsSync(outFile)) {
fs.rmSync(outFile);
}
const result = await compiler.run(infile, outFile);
assert.isNotNull(result);
assert.isTrue(await compiler.write(result.artifacts));
assert(fs.existsSync(outFile));
const outfileData = fs.readFileSync(outFile);
@ -54,7 +60,7 @@ describe('Compiler class', function() {
it('should build all baseline fixtures', async function() {
const compiler = new KmnCompiler();
const callbacks = new TestCompilerCallbacks();
assert(await compiler.init(callbacks));
assert(await compiler.init(callbacks, {saveDebug: true, shouldAddCompilerVersion: false}));
assert(compiler.verifyInitialized());
const files = fs.readdirSync(baselineDir);
@ -64,7 +70,13 @@ describe('Compiler class', function() {
const infile = baselineDir + file.replace(/x$/, 'n');
const outFile = __dirname + '/' + file;
assert(compiler.run(infile, {saveDebug: true, outFile, shouldAddCompilerVersion: false}));
if(fs.existsSync(outFile)) {
fs.rmSync(outFile);
}
const result = await compiler.run(infile, outFile);
assert.isNotNull(result);
assert.isTrue(await compiler.write(result.artifacts));
assert(fs.existsSync(outFile));
const outfileData = fs.readFileSync(outFile);
@ -78,7 +90,10 @@ describe('Compiler class', function() {
it('should compile a keyboard with visual keyboard', async function() {
const compiler = new KmnCompiler();
const callbacks = new TestCompilerCallbacks();
assert.isTrue(await compiler.init(callbacks));
assert.isTrue(await compiler.init(callbacks, {
saveDebug: true,
shouldAddCompilerVersion: false,
}));
assert.isTrue(compiler.verifyInitialized());
const fixtureDir = keyboardsDir + 'caps_lock_layer_3620/'
@ -89,11 +104,17 @@ describe('Compiler class', function() {
const resultingKmxfile = __dirname + '/caps_lock_layer_3620.kmx';
const resultingKvkfile = __dirname + '/caps_lock_layer_3620.kvk';
assert.isTrue(compiler.run(infile, {
saveDebug: true,
shouldAddCompilerVersion: false,
outFile: resultingKmxfile,
}));
if(fs.existsSync(resultingKmxfile)) {
fs.rmSync(resultingKmxfile);
}
if(fs.existsSync(resultingKvkfile)) {
fs.rmSync(resultingKvkfile);
}
const result = await compiler.run(infile, resultingKmxfile);
assert.isNotNull(result);
assert.isTrue(await compiler.write(result.artifacts));
assert.isTrue(fs.existsSync(resultingKmxfile));
assert.isTrue(fs.existsSync(resultingKvkfile));

10
developer/src/kmc-kmn/test/test-features.ts
View file

@ -5,14 +5,14 @@ import { TestCompilerCallbacks } from '@keymanapp/developer-test-helpers';
import { makePathToFixture } from './helpers/index.js';
import { KMX, KmxFileReader } from '@keymanapp/common-types';
describe('Keyboard compiler features', async function() {
describe('Keyboard compiler features', function() {
let compiler: KmnCompiler = null;
let callbacks: TestCompilerCallbacks = null;
this.beforeAll(async function() {
compiler = new KmnCompiler();
callbacks = new TestCompilerCallbacks();
assert(await compiler.init(callbacks));
assert(await compiler.init(callbacks, {saveDebug: true}));
assert(compiler.verifyInitialized());
});
@ -29,15 +29,15 @@ describe('Keyboard compiler features', async function() {
];
for(const v of versions) {
it(`should build a version ${v[0]} keyboard`, function() {
it(`should build a version ${v[0]} keyboard`, async function() {
const fixtureName = makePathToFixture('features', `version_${v[1]}.kmn`);
const result = compiler.runCompiler(fixtureName, {outFile: `version_${v[1]}.kmx`, saveDebug: true});
const result = await compiler.run(fixtureName, `version_${v[1]}.kmx`);
if(result === null) callbacks.printMessages();
assert.isNotNull(result);
const reader = new KmxFileReader();
const keyboard = reader.read(result.kmx.data);
const keyboard = reader.read(result.artifacts.kmx.data);
assert.equal(keyboard.fileVersion, v[2]);
});
}

4
developer/src/kmc-kmn/test/test-messages.ts
View file

@ -23,13 +23,13 @@ describe('CompilerMessages', function () {
callbacks.clear();
const compiler = new KmnCompiler();
assert(await compiler.init(callbacks));
assert(await compiler.init(callbacks, {saveDebug: true, shouldAddCompilerVersion: false}));
assert(compiler.verifyInitialized());
const kmnPath = makePathToFixture(...fixture);
// Note: throwing away compile results (just to memory)
compiler.runCompiler(kmnPath, {saveDebug: true, shouldAddCompilerVersion: false});
await compiler.run(kmnPath, null);
if(messageId) {
assert.isTrue(callbacks.hasMessage(messageId), `messageId ${messageId.toString(16)} not generated, instead got: `+JSON.stringify(callbacks.messages,null,2));

10
developer/src/kmc-kmn/test/test-wasm-uset.ts
View file

@ -25,14 +25,14 @@ describe('Compiler UnicodeSet function', function() {
it('should start', async function() {
const compiler = new KmnCompiler();
const callbacks = new TestCompilerCallbacks();
assert(await compiler.init(callbacks));
assert(await compiler.init(callbacks, null));
assert(compiler.verifyInitialized());
});
it('should compile a basic uset', async function() {
const compiler = new KmnCompiler();
const callbacks = new TestCompilerCallbacks();
assert(await compiler.init(callbacks));
assert(await compiler.init(callbacks, null));
assert(compiler.verifyInitialized());
const pat = "[abc]";
@ -50,7 +50,7 @@ describe('Compiler UnicodeSet function', function() {
it('should compile a more complex uset', async function() {
const compiler = new KmnCompiler();
const callbacks = new TestCompilerCallbacks();
assert(await compiler.init(callbacks));
assert(await compiler.init(callbacks, null));
assert(compiler.verifyInitialized());
const pat = "[[🙀A-C]-[CB]]";
@ -70,7 +70,7 @@ describe('Compiler UnicodeSet function', function() {
it('should compile an even more complex uset', async function() {
const compiler = new KmnCompiler();
const callbacks = new TestCompilerCallbacks();
assert(await compiler.init(callbacks));
assert(await compiler.init(callbacks, null));
assert(compiler.verifyInitialized());
const pat = "[\\u{10FFFD}\\u{2019}\\u{22}\\u{a}\\u{ead}\\u{1F640}]";
@ -97,7 +97,7 @@ describe('Compiler UnicodeSet function', function() {
it('should fail in various ways', async function() {
const compiler = new KmnCompiler();
const callbacks = new TestCompilerCallbacks();
assert(await compiler.init(callbacks));
assert(await compiler.init(callbacks, null));
assert(compiler.verifyInitialized());
// map from string to failing error
const failures = {

96
developer/src/kmc-ldml/src/compiler/compiler.ts
View file

@ -1,4 +1,4 @@
import { LDMLKeyboardXMLSourceFileReader, LDMLKeyboard, KMXPlus, CompilerCallbacks, LDMLKeyboardTestDataXMLSourceFile, UnicodeSetParser } from '@keymanapp/common-types';
import { LDMLKeyboardXMLSourceFileReader, LDMLKeyboard, KMXPlus, CompilerCallbacks, LDMLKeyboardTestDataXMLSourceFile, UnicodeSetParser, KeymanCompiler, KeymanCompilerResult, KeymanCompilerArtifacts, defaultCompilerOptions, KMXBuilder, KvkFileWriter, KeymanCompilerArtifactOptional } from '@keymanapp/common-types';
import { LdmlCompilerOptions } from './ldml-compiler-options.js';
import { CompilerMessages } from './messages.js';
import { BkspCompiler, TranCompiler } from './tran.js';
@ -16,6 +16,9 @@ import KMXPlusFile = KMXPlus.KMXPlusFile;
import DependencySections = KMXPlus.DependencySections;
import { SectionIdent, constants } from '@keymanapp/ldml-keyboard-constants';
import { KmnCompiler } from '@keymanapp/kmc-kmn';
import { KMXPlusMetadataCompiler } from './metadata-compiler.js';
import { LdmlKeyboardVisualKeyboardCompiler } from './visual-keyboard-compiler.js';
import { LdmlKeyboardKeymanWebCompiler } from './keymanweb-compiler.js';
export const SECTION_COMPILERS = [
// These are in dependency order.
@ -37,18 +40,93 @@ export const SECTION_COMPILERS = [
TranCompiler,
];
export class LdmlKeyboardCompiler {
private readonly callbacks: CompilerCallbacks;
private readonly options: LdmlCompilerOptions;
export interface LdmlKeyboardCompilerArtifacts extends KeymanCompilerArtifacts {
kmx?: KeymanCompilerArtifactOptional;
kvk?: KeymanCompilerArtifactOptional;
js?: KeymanCompilerArtifactOptional;
};
export interface LdmlKeyboardCompilerResult extends KeymanCompilerResult {
artifacts: LdmlKeyboardCompilerArtifacts;
};
export class LdmlKeyboardCompiler implements KeymanCompiler {
private callbacks: CompilerCallbacks;
private options: LdmlCompilerOptions;
// uset parser
private usetparser?: UnicodeSetParser = undefined;
constructor (callbacks: CompilerCallbacks, options: LdmlCompilerOptions) {
this.options = {
...options
};
async init(callbacks: CompilerCallbacks, options: LdmlCompilerOptions): Promise<boolean> {
this.options = {...options};
this.callbacks = callbacks;
return true;
}
async run(inputFilename: string, outputFilename?: string): Promise<LdmlKeyboardCompilerResult> {
let compilerOptions: LdmlCompilerOptions = {
...defaultCompilerOptions,
...this.options,
};
let source = this.load(inputFilename);
if (!source) {
return null;
}
let kmx = await this.compile(source);
if (!kmx) {
return null;
}
// In order for the KMX file to be loaded by non-KMXPlus components, it is helpful
// to duplicate some of the metadata
KMXPlusMetadataCompiler.addKmxMetadata(kmx.kmxplus, kmx.keyboard, compilerOptions);
// Use the builder to generate the binary output file
const builder = new KMXBuilder(kmx, compilerOptions.saveDebug);
const kmx_binary = builder.compile();
const vkcompiler = new LdmlKeyboardVisualKeyboardCompiler(this.callbacks);
const vk = vkcompiler.compile(source);
const writer = new KvkFileWriter();
const kvk_binary = writer.write(vk);
// Note: we could have a step of generating source files here
// KvksFileWriter()...
// const tlcompiler = new kmc.TouchLayoutCompiler();
// const tl = tlcompiler.compile(source);
// const tlwriter = new TouchLayoutFileWriter();
const kmwcompiler = new LdmlKeyboardKeymanWebCompiler(this.callbacks, compilerOptions);
const kmw_string = kmwcompiler.compile(inputFilename, source);
const encoder = new TextEncoder();
const kmw_binary = encoder.encode(kmw_string);
outputFilename = outputFilename ?? inputFilename.replace(/\.xml$/, '.kmx');
return {
artifacts: {
kmx: { data: kmx_binary, filename: outputFilename },
kvk: { data: kvk_binary, filename: outputFilename.replace(/\.kmx$/, '.kvk') },
js: { data: kmw_binary, filename: outputFilename.replace(/\.kmx$/, '.js') },
}
};
}
async write(artifacts: LdmlKeyboardCompilerArtifacts): Promise<boolean> {
if(artifacts.kmx) {
this.callbacks.fs.writeFileSync(artifacts.kmx.filename, artifacts.kmx.data);
}
if(artifacts.kvk) {
this.callbacks.fs.writeFileSync(artifacts.kvk.filename, artifacts.kvk.data);
}
if(artifacts.js) {
this.callbacks.fs.writeFileSync(artifacts.js.filename, artifacts.js.data);
}
return true;
}
/**
@ -59,7 +137,7 @@ export class LdmlKeyboardCompiler {
if (this.usetparser === undefined) {
// initialize
const compiler = new KmnCompiler();
const ok = await compiler.init(this.callbacks);
const ok = await compiler.init(this.callbacks, null);
if (ok) {
this.usetparser = compiler;
} else {

13
developer/src/kmc-ldml/test/helpers/index.ts
View file

@ -112,14 +112,16 @@ async function loadDepsFor(sections: DependencySections, parentCompiler: Section
}
}
export function loadTestdata(inputFilename: string, options: LdmlCompilerOptions) : LDMLKeyboardTestDataXMLSourceFile {
const k = new LdmlKeyboardCompiler(compilerTestCallbacks, options);
export async function loadTestData(inputFilename: string, options: LdmlCompilerOptions) : Promise<LDMLKeyboardTestDataXMLSourceFile> {
const k = new LdmlKeyboardCompiler();
assert.isTrue(await k.init(compilerTestCallbacks, options));
const source = k.loadTestData(inputFilename);
return source;
}
export async function compileKeyboard(inputFilename: string, options: LdmlCompilerOptions, validateMessages?: CompilerEvent[], expectFailValidate?: boolean, compileMessages?: CompilerEvent[]): Promise<KMXPlusFile> {
const k = new LdmlKeyboardCompiler(compilerTestCallbacks, options);
const k = new LdmlKeyboardCompiler();
assert.isTrue(await k.init(compilerTestCallbacks, options));
const source = k.load(inputFilename);
checkMessages();
assert.isNotNull(source, 'k.load should not have returned null');
@ -151,7 +153,8 @@ export async function compileKeyboard(inputFilename: string, options: LdmlCompil
}
export async function compileVisualKeyboard(inputFilename: string, options: LdmlCompilerOptions): Promise<VisualKeyboard.VisualKeyboard> {
const k = new LdmlKeyboardCompiler(compilerTestCallbacks, options);
const k = new LdmlKeyboardCompiler();
assert.isTrue(await k.init(compilerTestCallbacks, options));
const source = k.load(inputFilename);
checkMessages();
assert.isNotNull(source, 'k.load should not have returned null');
@ -254,7 +257,7 @@ async function getTestUnicodeSetParser(callbacks: CompilerCallbacks): Promise<Un
// for tests, just create a new one
// see LdmlKeyboardCompiler.getUsetParser()
const compiler = new KmnCompiler();
const ok = await compiler.init(callbacks);
const ok = await compiler.init(callbacks, null);
assert.ok(ok, `Could not initialize KmnCompiler (UnicodeSetParser), see callback messages`);
if (ok) {
return compiler;

27
developer/src/kmc-ldml/test/test-compiler-e2e.ts
View file

@ -12,7 +12,7 @@ describe('compiler-tests', function() {
before(function() {
compilerTestCallbacks.clear();
});
it('should-build-fixtures', async function() {
// Let's build basic.xml
// It should match basic.kmx (built from basic.txt)
@ -36,33 +36,38 @@ describe('compiler-tests', function() {
assert.deepEqual<Uint8Array>(code, expected);
});
it('should handle non existent files', () => {
it('should handle non existent files', async () => {
const filename = 'DOES_NOT_EXIST.xml';
const k = new LdmlKeyboardCompiler(compilerTestCallbacks, { ...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false });
const k = new LdmlKeyboardCompiler();
await k.init(compilerTestCallbacks, { ...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false });
const source = k.load(filename);
assert.notOk(source, `Trying to load(${filename})`);
});
it('should handle unparseable files', () => {
it('should handle unparseable files', async () => {
const filename = makePathToFixture('basic-kvk.txt'); // not an .xml file
const k = new LdmlKeyboardCompiler(compilerTestCallbacks, { ...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false });
const k = new LdmlKeyboardCompiler();
await k.init(compilerTestCallbacks, { ...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false });
const source = k.load(filename);
assert.notOk(source, `Trying to load(${filename})`);
});
it('should handle not-valid files', () => {
it('should handle not-valid files', async () => {
const filename = makePathToFixture('test-fr.xml'); // not a keyboard .xml file
const k = new LdmlKeyboardCompiler(compilerTestCallbacks, { ...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false });
const k = new LdmlKeyboardCompiler();
await k.init(compilerTestCallbacks, { ...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false });
const source = k.load(filename);
assert.notOk(source, `Trying to load(${filename})`);
});
it('should handle non existent test files', () => {
it('should handle non existent test files', async () => {
const filename = 'DOES_NOT_EXIST.xml';
const k = new LdmlKeyboardCompiler(compilerTestCallbacks, { ...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false });
const k = new LdmlKeyboardCompiler();
await k.init(compilerTestCallbacks, { ...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false });
const source = k.loadTestData(filename);
assert.notOk(source, `Trying to loadTestData(${filename})`);
});
it('should handle unparseable test files', () => {
it('should handle unparseable test files', async () => {
const filename = makePathToFixture('basic-kvk.txt'); // not an .xml file
const k = new LdmlKeyboardCompiler(compilerTestCallbacks, { ...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false });
const k = new LdmlKeyboardCompiler();
await k.init(compilerTestCallbacks, { ...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false });
const source = k.load(filename);
assert.notOk(source, `Trying to loadTestData(${filename})`);
});

3
developer/src/kmc-ldml/test/test-keymanweb-compiler.ts
View file

@ -16,7 +16,8 @@ describe('LdmlKeyboardKeymanWebCompiler', function() {
// Load input data; we'll use the LDML keyboard compiler loader to save us
// effort here
const k = new LdmlKeyboardCompiler(compilerTestCallbacks, {...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false});
const k = new LdmlKeyboardCompiler();
await k.init(compilerTestCallbacks, {...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false});
const source = k.load(inputFilename);
checkMessages();
assert.isNotNull(source, 'k.load should not have returned null');

4
developer/src/kmc-ldml/test/test-testdata-e2e.ts
View file

@ -1,7 +1,7 @@
import { readFileSync } from 'fs';
import 'mocha';
import {assert} from 'chai';
import {compilerTestOptions, loadTestdata, makePathToFixture} from './helpers/index.js';
import {compilerTestOptions, loadTestData, makePathToFixture} from './helpers/index.js';
describe('testdata-tests', function() {
this.slow(500); // 0.5 sec -- json schema validation takes a while
@ -15,7 +15,7 @@ describe('testdata-tests', function() {
const jsonFilename = makePathToFixture('test-fr.json');
// Compile the keyboard
const testData = loadTestdata(inputFilename, {...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false});
const testData = await loadTestData(inputFilename, {...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false});
assert.isNotNull(testData);
const jsonData = JSON.parse(readFileSync(jsonFilename, 'utf-8'));

50
developer/src/kmc-model-info/src/model-info-compiler.ts
View file

@ -5,7 +5,7 @@
import { minKeymanVersion } from "./min-keyman-version.js";
import { ModelInfoFile } from "./model-info-file.js";
import { CompilerCallbacks, KmpJsonFile } from "@keymanapp/common-types";
import { CompilerCallbacks, CompilerOptions, KeymanCompiler, KeymanCompilerArtifact, KeymanCompilerArtifacts, KeymanCompilerResult, KmpJsonFile } from "@keymanapp/common-types";
import { ModelInfoCompilerMessages } from "./messages.js";
import { validateMITLicense } from "@keymanapp/developer-utils";
@ -39,8 +39,30 @@ export class ModelInfoSources {
};
/* c8 ignore stop */
export class ModelInfoCompiler {
constructor(private callbacks: CompilerCallbacks) {
export interface ModelInfoCompilerOptions extends CompilerOptions {
sources: ModelInfoSources;
};
export interface ModelInfoCompilerArtifacts extends KeymanCompilerArtifacts {
model_info: KeymanCompilerArtifact;
};
export interface ModelInfoCompilerResult extends KeymanCompilerResult {
artifacts: ModelInfoCompilerArtifacts;
};
export class ModelInfoCompiler implements KeymanCompiler {
private callbacks: CompilerCallbacks;
private options: ModelInfoCompilerOptions;
constructor() {
}
public async init(callbacks: CompilerCallbacks, options: ModelInfoCompilerOptions): Promise<boolean> {
this.callbacks = callbacks;
this.options = {...options};
return true;
}
/**
@ -51,9 +73,8 @@ export class ModelInfoCompiler {
*
* @param sources Details on files from which to extract additional metadata
*/
writeModelMetadataFile(
sources: ModelInfoSources
): Uint8Array {
public async run(inputFilename: string, outputFilename?: string): Promise<ModelInfoCompilerResult> {
const sources = this.options.sources;
/*
* Model info looks like this:
@ -171,7 +192,22 @@ export class ModelInfoCompiler {
}
const jsonOutput = JSON.stringify(model_info, null, 2);
return new TextEncoder().encode(jsonOutput);
const data = new TextEncoder().encode(jsonOutput);
const result: ModelInfoCompilerResult = {
artifacts: {
model_info: {
data,
filename: outputFilename ?? inputFilename.replace(/\.kpj$/, '.model_info')
}
}
};
return result;
}
public async write(artifacts: ModelInfoCompilerArtifacts): Promise<boolean> {
this.callbacks.fs.writeFileSync(artifacts.model_info.filename, artifacts.model_info.data);
return true;
}
private isLicenseMIT(filename: string) {

19
developer/src/kmc-model-info/test/test-model-info-compiler.ts
View file

@ -13,16 +13,18 @@ beforeEach(function() {
});
describe('model-info-compiler', function () {
it('compile a .model_info file correctly', function() {
it('compile a .model_info file correctly', async function() {
const kpjFilename = makePathToFixture('sil.cmo.bw', 'sil.cmo.bw.model.kpj');
const kpsFilename = makePathToFixture('sil.cmo.bw', 'source', 'sil.cmo.bw.model.kps');
const kmpFileName = makePathToFixture('sil.cmo.bw', 'build', 'sil.cmo.bw.model.kmp');
const buildModelInfoFilename = makePathToFixture('sil.cmo.bw', 'build', 'sil.cmo.bw.model_info');
const kmpCompiler = new KmpCompiler(callbacks);
const kmpCompiler = new KmpCompiler();
assert.isTrue(await kmpCompiler.init(callbacks, {}));
const kmpJsonData = kmpCompiler.transformKpsToKmpObject(kpsFilename);
const modelFileName = makePathToFixture('sil.cmo.bw', 'build', 'sil.cmo.bw.model.js');
const data = (new ModelInfoCompiler(callbacks)).writeModelMetadataFile({
const sources = {
kmpFileName,
kmpJsonData,
model_id: 'sil.cmo.bw',
@ -30,13 +32,16 @@ describe('model-info-compiler', function () {
sourcePath: 'release/sil/sil.cmo.bw',
kpsFilename,
forPublishing: true,
});
if(data == null) {
};
const compiler = new ModelInfoCompiler();
assert.isTrue(await compiler.init(callbacks, {sources}));
const result = await compiler.run(kpjFilename, null);
if(result == null) {
callbacks.printMessages();
}
assert.isNotNull(data);
assert.isNotNull(result);
const actual = JSON.parse(new TextDecoder().decode(data));
const actual = JSON.parse(new TextDecoder().decode(result.artifacts.model_info.data));
let expected = JSON.parse(fs.readFileSync(buildModelInfoFilename, 'utf-8'));
// `lastModifiedDate` is dependent on time of run (not worth mocking)

92
developer/src/kmc-model/src/lexical-model-compiler.ts
View file

@ -2,20 +2,102 @@
lexical-model-compiler.ts: base file for lexical model compiler.
*/
import * as ts from "typescript";
import ts from "typescript";
import { createTrieDataStructure } from "./build-trie.js";
import { ModelDefinitions } from "./model-definitions.js";
import {decorateWithJoin} from "./join-word-breaker-decorator.js";
import {decorateWithScriptOverrides} from "./script-overrides-decorator.js";
import { LexicalModelSource, WordBreakerSpec, SimpleWordBreakerSpec } from "./lexical-model.js";
import { ModelCompilerError, ModelCompilerMessages } from "./model-compiler-errors.js";
import { ModelCompilerError, ModelCompilerMessageContext, ModelCompilerMessages } from "./model-compiler-errors.js";
import { callbacks, setCompilerCallbacks } from "./compiler-callbacks.js";
import { CompilerCallbacks } from "@keymanapp/common-types";
import { CompilerCallbacks, CompilerOptions, KeymanCompiler, KeymanCompilerArtifact, KeymanCompilerArtifacts, KeymanCompilerResult } from "@keymanapp/common-types";
export default class LexicalModelCompiler {
/**
* An ECMAScript module as emitted by the TypeScript compiler.
*/
interface ES2015Module {
/** This is always true. */
__esModule: boolean;
'default'?: unknown;
};
constructor(callbacks: CompilerCallbacks) {
export interface LexicalModelCompilerArtifacts extends KeymanCompilerArtifacts {
js: KeymanCompilerArtifact;
};
export interface LexicalModelCompilerResult extends KeymanCompilerResult {
artifacts: LexicalModelCompilerArtifacts;
};
export class LexicalModelCompiler implements KeymanCompiler {
async init(callbacks: CompilerCallbacks, _options: CompilerOptions): Promise<boolean> {
setCompilerCallbacks(callbacks);
return true;
}
async run(inputFilename: string, outputFilename?: string): Promise<LexicalModelCompilerResult> {
try {
let modelSource = this.loadFromFilename(inputFilename);
let containingDirectory = callbacks.path.dirname(inputFilename);
let code = this.generateLexicalModelCode('<unknown>', modelSource, containingDirectory);
const result: LexicalModelCompilerResult = {
artifacts: {
js: {
data: new TextEncoder().encode(code),
filename: outputFilename ?? inputFilename.replace(/\.model\.ts$/, '.model.js')
}
}
}
return result;
} catch(e) {
callbacks.reportMessage(
e instanceof ModelCompilerError
? e.event
: ModelCompilerMessages.Fatal_UnexpectedException({e:e})
);
return null;
}
}
async write(artifacts: LexicalModelCompilerArtifacts): Promise<boolean> {
callbacks.fs.writeFileSync(artifacts.js.filename, artifacts.js.data);
return true;
}
/**
* Loads a lexical model's source module from the given filename.
*
* @param filename path to the model source file.
*/
public loadFromFilename(filename: string): LexicalModelSource {
let sourceCode = new TextDecoder().decode(callbacks.loadFile(filename));
// Compile the module to JavaScript code.
// NOTE: transpile module does a very simple TS to JS compilation.
// It DOES NOT check for types!
let compilationOutput = ts.transpile(sourceCode, {
// Our runtime only supports ES3 with Node/CommonJS modules on Android 5.0.
// When we drop Android 5.0 support, we can update this to a `ScriptTarget`
// matrix against target version of Keyman, here and in
// lexical-model-compiler.ts.
target: ts.ScriptTarget.ES3,
module: ts.ModuleKind.CommonJS,
});
// Turn the module into a function in which we can inject a global.
let moduleCode = '(function(exports){' + compilationOutput + '})';
// Run the module; its exports will be assigned to `moduleExports`.
let moduleExports: Partial<ES2015Module> = {};
let module = eval(moduleCode);
module(moduleExports);
if (!moduleExports['__esModule'] || !moduleExports['default']) {
ModelCompilerMessageContext.filename = filename;
throw new ModelCompilerError(ModelCompilerMessages.Error_NoDefaultExport());
}
return moduleExports['default'] as LexicalModelSource;
}
/**

80
developer/src/kmc-model/src/main.ts
View file

@ -1,79 +1 @@
import { CompilerCallbacks } from '@keymanapp/common-types';
import ts from 'typescript';
import { setCompilerCallbacks } from './compiler-callbacks.js';
import LexicalModelCompiler from './lexical-model-compiler.js';
import { LexicalModelSource } from './lexical-model.js';
import { ModelCompilerError, ModelCompilerMessageContext, ModelCompilerMessages } from './model-compiler-errors.js';
export { default as LexicalModelCompiler } from './lexical-model-compiler.js';
/**
* Compiles a model.ts file, using paths relative to its location.
*
* @param filename path to model.ts source.
* @return model source code, or null on error
*/
export function compileModel(filename: string, callbacks: CompilerCallbacks): string {
setCompilerCallbacks(callbacks);
try {
let modelSource = loadFromFilename(filename, callbacks);
let containingDirectory = callbacks.path.dirname(filename);
return (new LexicalModelCompiler(callbacks))
.generateLexicalModelCode('<unknown>', modelSource, containingDirectory);
} catch(e) {
callbacks.reportMessage(
e instanceof ModelCompilerError
? e.event
: ModelCompilerMessages.Fatal_UnexpectedException({e:e})
);
}
return null;
}
/**
* An ECMAScript module as emitted by the TypeScript compiler.
*/
interface ES2015Module {
/** This is always true. */
__esModule: boolean;
'default'?: unknown;
}
/**
* Loads a lexical model's source module from the given filename.
*
* @param filename path to the model source file.
*/
export function loadFromFilename(filename: string, callbacks: CompilerCallbacks): LexicalModelSource {
setCompilerCallbacks(callbacks);
let sourceCode = new TextDecoder().decode(callbacks.loadFile(filename));
// Compile the module to JavaScript code.
// NOTE: transpile module does a very simple TS to JS compilation.
// It DOES NOT check for types!
let compilationOutput = ts.transpile(sourceCode, {
// Our runtime only supports ES3 with Node/CommonJS modules on Android 5.0.
// When we drop Android 5.0 support, we can update this to a `ScriptTarget`
// matrix against target version of Keyman, here and in
// lexical-model-compiler.ts.
target: ts.ScriptTarget.ES3,
module: ts.ModuleKind.CommonJS,
});
// Turn the module into a function in which we can inject a global.
let moduleCode = '(function(exports){' + compilationOutput + '})';
// Run the module; its exports will be assigned to `moduleExports`.
let moduleExports: Partial<ES2015Module> = {};
let module = eval(moduleCode);
module(moduleExports);
if (!moduleExports['__esModule'] || !moduleExports['default']) {
ModelCompilerMessageContext.filename = filename;
throw new ModelCompilerError(ModelCompilerMessages.Error_NoDefaultExport());
}
return moduleExports['default'] as LexicalModelSource;
}
export { LexicalModelCompiler } from './lexical-model-compiler.js';

32
developer/src/kmc-model/test/test-compile-model-with-pseudoclosure.ts
View file

@ -1,4 +1,4 @@
import LexicalModelCompiler from '../src/lexical-model-compiler.js';
import { LexicalModelCompiler } from '../src/lexical-model-compiler.js';
import {assert} from 'chai';
import 'mocha';
@ -26,8 +26,9 @@ describe('LexicalModelCompiler - pseudoclosure compilation + use', function () {
}
};
it('variant 1: applyCasing prepends symbols, searchTermToKey removes them', function() {
let compiler = new LexicalModelCompiler(callbacks);
it('variant 1: applyCasing prepends symbols, searchTermToKey removes them', async function() {
let compiler = new LexicalModelCompiler();
assert.isTrue(await compiler.init(callbacks, null));
let code = compiler.generateLexicalModelCode(MODEL_ID, {
format: 'trie-1.0',
sources: ['wordlist.tsv'],
@ -80,8 +81,9 @@ describe('LexicalModelCompiler - pseudoclosure compilation + use', function () {
assert.isNotNull(compilation.exportedModel);
});
it('variant 2: applyCasing prepends symbols, searchTermToKey keeps them', function() {
let compiler = new LexicalModelCompiler(callbacks);
it('variant 2: applyCasing prepends symbols, searchTermToKey keeps them', async function() {
let compiler = new LexicalModelCompiler();
assert.isTrue(await compiler.init(callbacks, null));
let code = compiler.generateLexicalModelCode(MODEL_ID, {
format: 'trie-1.0',
sources: ['wordlist.tsv'],
@ -124,8 +126,9 @@ describe('LexicalModelCompiler - pseudoclosure compilation + use', function () {
assert.isNotNull(compilation.exportedModel);
});
it('variant 3: applyCasing prepends symbols, default searchTermToKey', function() {
let compiler = new LexicalModelCompiler(callbacks);
it('variant 3: applyCasing prepends symbols, default searchTermToKey', async function() {
let compiler = new LexicalModelCompiler();
assert.isTrue(await compiler.init(callbacks, null));
let code = compiler.generateLexicalModelCode(MODEL_ID, {
format: 'trie-1.0',
sources: ['wordlist.tsv'],
@ -164,8 +167,9 @@ describe('LexicalModelCompiler - pseudoclosure compilation + use', function () {
});
describe('relying on default applyCasing + searchTermToKey', function() {
it('languageUsesCasing: true', function() {
let compiler = new LexicalModelCompiler(callbacks);
it('languageUsesCasing: true', async function() {
let compiler = new LexicalModelCompiler();
assert.isTrue(await compiler.init(callbacks, null));
let code = compiler.generateLexicalModelCode(MODEL_ID, {
format: 'trie-1.0',
sources: ['wordlist.tsv'],
@ -188,8 +192,9 @@ describe('LexicalModelCompiler - pseudoclosure compilation + use', function () {
assert.isNotNull(compilation.exportedModel);
});
it('languageUsesCasing: false', function() {
let compiler = new LexicalModelCompiler(callbacks);
it('languageUsesCasing: false', async function() {
let compiler = new LexicalModelCompiler();
assert.isTrue(await compiler.init(callbacks, null));
let code = compiler.generateLexicalModelCode(MODEL_ID, {
format: 'trie-1.0',
sources: ['wordlist.tsv'],
@ -213,8 +218,9 @@ describe('LexicalModelCompiler - pseudoclosure compilation + use', function () {
assert.isNotNull(compilation.exportedModel);
});
it('languageUsesCasing: undefined', function() {
let compiler = new LexicalModelCompiler(callbacks);
it('languageUsesCasing: undefined', async function() {
let compiler = new LexicalModelCompiler();
assert.isTrue(await compiler.init(callbacks, null));
let code = compiler.generateLexicalModelCode(MODEL_ID, {
format: 'trie-1.0',
sources: ['wordlist.tsv']

13
developer/src/kmc-model/test/test-compile-model.ts
View file

@ -1,12 +1,12 @@
import 'mocha';
import {assert} from 'chai';
import {compileModel} from '../src/main.js';
import { LexicalModelCompiler } from '../src/main.js';
import {makePathToFixture, compileModelSourceCode, CompilationResult} from './helpers/index.js';
import { TestCompilerCallbacks } from '@keymanapp/developer-test-helpers';
import { KeymanFileTypes } from '@keymanapp/common-types';
describe('compileModel', function () {
describe('LexicalModelCompiler', function () {
let callbacks = new TestCompilerCallbacks();
// Try to compile ALL of the correct models.
@ -22,9 +22,14 @@ describe('compileModel', function () {
for (let modelID of MODELS) {
let modelPath = makePathToFixture(modelID, modelID + KeymanFileTypes.Source.Model);
it(`should compile ${modelID}`, function () {
let code = compileModel(modelPath, callbacks);
it(`should compile ${modelID}`, async function () {
const compiler = new LexicalModelCompiler();
assert.isTrue(await compiler.init(callbacks, null));
const result = await compiler.run(modelPath, null);
callbacks.printMessages();
assert.isNotNull(result);
const decoder = new TextDecoder();
const code = decoder.decode(result.artifacts.js.data);
let r = compileModelSourceCode(code);
let compilation = r as CompilationResult;

27
developer/src/kmc-model/test/test-compile-trie.ts
View file

@ -1,4 +1,4 @@
import LexicalModelCompiler from '../src/lexical-model-compiler.js';
import { LexicalModelCompiler } from '../src/lexical-model-compiler.js';
import {assert} from 'chai';
import 'mocha';
@ -14,11 +14,12 @@ describe('LexicalModelCompiler', function () {
});
describe('#generateLexicalModelCode', function () {
it('should compile a trivial word list', function () {
it('should compile a trivial word list', async function () {
const MODEL_ID = 'example.qaa.trivial';
const PATH = makePathToFixture(MODEL_ID);
let compiler = new LexicalModelCompiler(callbacks);
let compiler = new LexicalModelCompiler();
assert.isTrue(await compiler.init(callbacks, null));
let code = compiler.generateLexicalModelCode(MODEL_ID, {
format: 'trie-1.0',
sources: ['wordlist.tsv']
@ -37,11 +38,12 @@ describe('LexicalModelCompiler', function () {
assert.match(code, /\bwordBreaker\b["']?:\s*wordBreakers\b/);
});
it('should compile a word list exported by Microsoft Excel', function () {
it('should compile a word list exported by Microsoft Excel', async function () {
const MODEL_ID = 'example.qaa.utf16le';
const PATH = makePathToFixture(MODEL_ID);
let compiler = new LexicalModelCompiler(callbacks);
let compiler = new LexicalModelCompiler();
assert.isTrue(await compiler.init(callbacks, null));
let code = compiler.generateLexicalModelCode(MODEL_ID, {
format: 'trie-1.0',
sources: ['wordlist.txt']
@ -58,11 +60,12 @@ describe('LexicalModelCompiler', function () {
});
});
it('should compile a word list with a custom word breaking function', function () {
it('should compile a word list with a custom word breaking function', async function () {
const MODEL_ID = 'example.qaa.trivial';
const PATH = makePathToFixture(MODEL_ID);
let compiler = new LexicalModelCompiler(callbacks);
let compiler = new LexicalModelCompiler();
assert.isTrue(await compiler.init(callbacks, null));
let code = compiler.generateLexicalModelCode(MODEL_ID, {
format: 'trie-1.0',
sources: ['wordlist.tsv'],
@ -81,11 +84,12 @@ describe('LexicalModelCompiler', function () {
assert.match(code, /\bwordBreaker\b["']?:\s+function\b/);
});
it('should not generate unpaired surrogate code units', function () {
it('should not generate unpaired surrogate code units', async function () {
const MODEL_ID = 'example.qaa.smp';
const PATH = makePathToFixture(MODEL_ID);
let compiler = new LexicalModelCompiler(callbacks);
let compiler = new LexicalModelCompiler();
assert.isTrue(await compiler.init(callbacks, null));
let code = compiler.generateLexicalModelCode(MODEL_ID, {
format: 'trie-1.0',
sources: ['wordlist.tsv']
@ -115,10 +119,11 @@ describe('LexicalModelCompiler', function () {
assert.match(code, /\btotalWeight\b["']?:\s*27596\b/);
});
it('should include the source code of its search term to key function', function () {
it('should include the source code of its search term to key function', async function () {
const MODEL_ID = 'example.qaa.trivial';
const PATH = makePathToFixture(MODEL_ID);
let compiler = new LexicalModelCompiler(callbacks);
let compiler = new LexicalModelCompiler();
assert.isTrue(await compiler.init(callbacks, null));
let code = compiler.generateLexicalModelCode(MODEL_ID, {
format: 'trie-1.0',
sources: ['wordlist.tsv']

7
developer/src/kmc-model/test/test-punctuation.ts
View file

@ -1,4 +1,4 @@
import LexicalModelCompiler from '../src/lexical-model-compiler.js';
import { LexicalModelCompiler } from '../src/lexical-model-compiler.js';
import {assert} from 'chai';
import 'mocha';
@ -10,10 +10,11 @@ describe('LexicalModelCompiler', function () {
const MODEL_ID = 'example.qaa.trivial';
const PATH = makePathToFixture(MODEL_ID);
it('should compile punctuation into the generated code', function () {
it('should compile punctuation into the generated code', async function () {
const callbacks = new TestCompilerCallbacks();
let compiler = new LexicalModelCompiler(callbacks);
let compiler = new LexicalModelCompiler();
assert.isTrue(await compiler.init(callbacks, null));
let code = compiler.generateLexicalModelCode(MODEL_ID, {
format: 'trie-1.0',
sources: ['wordlist.tsv'],

70
developer/src/kmc-package/src/compiler/kmp-compiler.ts
View file

@ -2,7 +2,7 @@ import * as xml2js from 'xml2js';
import JSZip from 'jszip';
import KEYMAN_VERSION from "@keymanapp/keyman-version";
import { KmpJsonFile, KpsFile, SchemaValidators, CompilerCallbacks, KeymanFileTypes, KvkFile } from '@keymanapp/common-types';
import { KmpJsonFile, KpsFile, SchemaValidators, CompilerCallbacks, KeymanFileTypes, KvkFile, KeymanCompiler, CompilerOptions, KeymanCompilerResult, KeymanCompilerArtifacts, KeymanCompilerArtifact } from '@keymanapp/common-types';
import { CompilerMessages } from './messages.js';
import { PackageMetadataCollector } from './package-metadata-collector.js';
import { KmpInfWriter } from './kmp-inf-writer.js';
@ -11,6 +11,7 @@ import { MIN_LM_FILEVERSION_KMP_JSON, PackageVersionValidator } from './package-
import { PackageKeyboardTargetValidator } from './package-keyboard-target-validator.js';
import { PackageMetadataUpdater } from './package-metadata-updater.js';
import { markdownToHTML } from './markdown.js';
import { PackageValidation } from './package-validation.js';
const KMP_JSON_FILENAME = 'kmp.json';
const KMP_INF_FILENAME = 'kmp.inf';
@ -20,9 +21,68 @@ const KMP_INF_FILENAME = 'kmp.inf';
// this filename for existing keyboard packages.
const WELCOME_HTM_FILENAME = 'welcome.htm';
export class KmpCompiler {
export interface KmpCompilerOptions extends CompilerOptions {
// Note: WindowsPackageInstallerCompilerOptions extends KmpCompilerOptions, so
// be careful when modifying this interface
};
constructor(private callbacks: CompilerCallbacks) {
export interface KmpCompilerArtifacts extends KeymanCompilerArtifacts {
kmp: KeymanCompilerArtifact;
};
export interface KmpCompilerResult extends KeymanCompilerResult {
artifacts: KmpCompilerArtifacts;
};
export class KmpCompiler implements KeymanCompiler {
private callbacks: CompilerCallbacks;
private options: KmpCompilerOptions;
public async init(callbacks: CompilerCallbacks, options: KmpCompilerOptions): Promise<boolean> {
this.callbacks = callbacks;
this.options = options ? {...options} : {};
return true;
}
public async run(inputFilename: string, outputFilename?: string): Promise<KmpCompilerResult> {
const kmpJsonData = this.transformKpsToKmpObject(inputFilename);
if(!kmpJsonData) {
return null;
}
//
// Validate the package file
//
const validation = new PackageValidation(this.callbacks, this.options);
if(!validation.validate(inputFilename, kmpJsonData)) {
return null;
}
//
// Build the .kmp package file
//
const data = await this.buildKmpFile(inputFilename, kmpJsonData);
if(!data) {
return null;
}
const result: KmpCompilerResult = {
artifacts: {
kmp: {
data,
filename: outputFilename ?? inputFilename.replace(/\.kps$/, '.kmp')
}
}
}
return result;
}
public async write(artifacts: KmpCompilerArtifacts): Promise<boolean> {
this.callbacks.fs.writeFileSync(artifacts.kmp.filename, artifacts.kmp.data);
return true;
}
public transformKpsToKmpObject(kpsFilename: string): KmpJsonFile.KmpJsonFile {
@ -346,7 +406,7 @@ export class KmpCompiler {
* @param kpsFilename - Filename of the kps, not read, used only for calculating relative paths
* @param kmpJsonData - The kmp.json Object
*/
public buildKmpFile(kpsFilename: string, kmpJsonData: KmpJsonFile.KmpJsonFile): Promise<string> {
public buildKmpFile(kpsFilename: string, kmpJsonData: KmpJsonFile.KmpJsonFile): Promise<Uint8Array> {
const zip = JSZip();
@ -438,7 +498,7 @@ export class KmpCompiler {
}
// Generate kmp file
return zip.generateAsync({type: 'binarystring', compression:'DEFLATE'});
return zip.generateAsync({type:'uint8array', compression:'DEFLATE'});
}
private buildKmpInf(data: KmpJsonFile.KmpJsonFile): Uint8Array {

53
developer/src/kmc-package/src/compiler/windows-package-installer-compiler.ts
View file

@ -11,9 +11,9 @@
*/
import JSZip from 'jszip';
import { CompilerCallbacks, KeymanFileTypes, KmpJsonFile, KpsFile } from "@keymanapp/common-types";
import { CompilerCallbacks, KeymanCompiler, KeymanCompilerArtifact, KeymanCompilerArtifacts, KeymanCompilerResult, KeymanFileTypes, KmpJsonFile, KpsFile } from "@keymanapp/common-types";
import KEYMAN_VERSION from "@keymanapp/keyman-version";
import { KmpCompiler } from "./kmp-compiler.js";
import { KmpCompiler, KmpCompilerOptions } from "./kmp-compiler.js";
import { CompilerMessages } from "./messages.js";
const SETUP_INF_FILENAME = 'setup.inf';
@ -30,15 +30,33 @@ export interface WindowsPackageInstallerSources {
startWithConfiguration: boolean;
};
export class WindowsPackageInstallerCompiler {
private kmpCompiler: KmpCompiler;
export interface WindowsPackageInstallerCompilerOptions extends KmpCompilerOptions {
sources: WindowsPackageInstallerSources;
}
constructor(private callbacks: CompilerCallbacks) {
this.kmpCompiler = new KmpCompiler(this.callbacks);
export interface WindowsPackageInstallerCompilerArtifacts extends KeymanCompilerArtifacts {
exe: KeymanCompilerArtifact;
};
export interface WindowsPackageInstallerCompilerResult extends KeymanCompilerResult {
artifacts: WindowsPackageInstallerCompilerArtifacts;
};
export class WindowsPackageInstallerCompiler implements KeymanCompiler {
private kmpCompiler: KmpCompiler;
private callbacks: CompilerCallbacks;
private options: WindowsPackageInstallerCompilerOptions;
async init(callbacks: CompilerCallbacks, options: WindowsPackageInstallerCompilerOptions): Promise<boolean> {
this.callbacks = callbacks;
this.options = {...options};
this.kmpCompiler = new KmpCompiler();
return await this.kmpCompiler.init(callbacks, options);
}
public async compile(kpsFilename: string, sources: WindowsPackageInstallerSources): Promise<Uint8Array> {
const kps = this.kmpCompiler.loadKpsFile(kpsFilename);
public async run(inputFilename: string, outputFilename?: string): Promise<WindowsPackageInstallerCompilerResult> {
const sources = this.options.sources;
const kps = this.kmpCompiler.loadKpsFile(inputFilename);
if(!kps) {
// errors will already have been reported by loadKpsFile
return null;
@ -64,7 +82,7 @@ export class WindowsPackageInstallerCompiler {
// Nor do we use the MSIOptions field.
// Build the zip
const zipBuffer = await this.buildZip(kps, kpsFilename, sources);
const zipBuffer = await this.buildZip(kps, inputFilename, sources);
if(!zipBuffer) {
// Error messages already reported by buildZip
return null;
@ -72,7 +90,22 @@ export class WindowsPackageInstallerCompiler {
// Build the sfx
const sfxBuffer = this.buildSfx(zipBuffer, sources);
return sfxBuffer;
const result: WindowsPackageInstallerCompilerResult = {
artifacts: {
exe: {
data: sfxBuffer,
filename: outputFilename ?? inputFilename.replace(/\.kps$/, '.exe')
}
}
};
return result;
}
public async write(artifacts: WindowsPackageInstallerCompilerArtifacts): Promise<boolean> {
this.callbacks.fs.writeFileSync(artifacts.exe.filename, artifacts.exe.data);
return true;
}
private async buildZip(kps: KpsFile.KpsFile, kpsFilename: string, sources: WindowsPackageInstallerSources): Promise<Uint8Array> {

1
developer/src/kmc-package/test/fixtures/invalid/example.qaa.sencoten.model.js vendored Normal file
View file

@ -0,0 +1 @@
// dummy file for unit tests

1
developer/src/kmc-package/test/fixtures/invalid/keyman.exe vendored Normal file
View file

@ -0,0 +1 @@
This is a dummy file for testing the unit tests

1
developer/src/kmc-package/test/fixtures/invalid/khmer_angkor.docx vendored Normal file
View file

@ -0,0 +1 @@
This is a sample text file

79
developer/src/kmc-package/test/test-messages.ts
View file

@ -4,7 +4,6 @@ import { TestCompilerCallbacks, verifyCompilerMessagesObject } from '@keymanapp/
import { CompilerMessages } from '../src/compiler/messages.js';
import { makePathToFixture } from './helpers/index.js';
import { KmpCompiler } from '../src/compiler/kmp-compiler.js';
import { PackageValidation } from '../src/compiler/package-validation.js';
import { CompilerErrorNamespace, CompilerOptions } from '@keymanapp/common-types';
const debug = false;
@ -20,24 +19,16 @@ describe('CompilerMessages', function () {
// Message tests
//
function testForMessage(context: Mocha.Context, fixture: string[], messageId?: number, options?: CompilerOptions) {
async function testForMessage(context: Mocha.Context, fixture: string[], messageId?: number, options?: CompilerOptions) {
context.timeout(10000);
callbacks.clear();
const kpsPath = makePathToFixture(...fixture);
const kmpCompiler = new KmpCompiler(callbacks);
const kmpCompiler = new KmpCompiler();
assert.isTrue(await kmpCompiler.init(callbacks, options ?? {}));
let kmpJson = kmpCompiler.transformKpsToKmpObject(kpsPath);
if(kmpJson && callbacks.messages.length == 0) {
const validator = new PackageValidation(callbacks, options ?? {});
validator.validate(kpsPath, kmpJson); // we'll ignore return value and rely on the messages
}
if(kmpJson && callbacks.messages.length == 0) {
// We'll try building the package if we have not yet received any messages
kmpCompiler.buildKmpFile(kpsPath, kmpJson)
}
await kmpCompiler.run(kpsPath);
if(debug) callbacks.printMessages();
@ -52,106 +43,106 @@ describe('CompilerMessages', function () {
// WARN_FileIsNotABinaryKvkFile
it('should generate WARN_FileIsNotABinaryKvkFile if a non-binary kvk file is included', async function() {
testForMessage(this, ['xml_kvk_file', 'source', 'xml_kvk_file.kps'], CompilerMessages.WARN_FileIsNotABinaryKvkFile);
await testForMessage(this, ['xml_kvk_file', 'source', 'xml_kvk_file.kps'], CompilerMessages.WARN_FileIsNotABinaryKvkFile);
});
it('should not warn if a binary kvk file is included', async function() {
testForMessage(this, ['binary_kvk_file', 'source', 'binary_kvk_file.kps']);
await testForMessage(this, ['binary_kvk_file', 'source', 'binary_kvk_file.kps']);
});
// ERROR_FollowKeyboardVersionNotAllowedForModelPackages
it('should generate ERROR_FollowKeyboardVersionNotAllowedForModelPackages if <FollowKeyboardVersion> is set for model packages', async function() {
testForMessage(this, ['invalid', 'followkeyboardversion.qaa.sencoten.model.kps'], CompilerMessages.ERROR_FollowKeyboardVersionNotAllowedForModelPackages);
await testForMessage(this, ['invalid', 'followkeyboardversion.qaa.sencoten.model.kps'], CompilerMessages.ERROR_FollowKeyboardVersionNotAllowedForModelPackages);
});
// ERROR_FollowKeyboardVersionButNoKeyboards
it('should generate ERROR_FollowKeyboardVersionButNoKeyboards if <FollowKeyboardVersion> is set for a package with no keyboards', async function() {
testForMessage(this, ['invalid', 'followkeyboardversion.empty.kps'], CompilerMessages.ERROR_FollowKeyboardVersionButNoKeyboards);
await testForMessage(this, ['invalid', 'followkeyboardversion.empty.kps'], CompilerMessages.ERROR_FollowKeyboardVersionButNoKeyboards);
});
// ERROR_KeyboardContentFileNotFound
it('should generate ERROR_KeyboardContentFileNotFound if a <Keyboard> is listed in a package but not found in <Files>', async function() {
testForMessage(this, ['invalid', 'keyboardcontentfilenotfound.kps'], CompilerMessages.ERROR_KeyboardContentFileNotFound);
await testForMessage(this, ['invalid', 'keyboardcontentfilenotfound.kps'], CompilerMessages.ERROR_KeyboardContentFileNotFound);
});
// ERROR_KeyboardFileNotValid
it('should generate ERROR_KeyboardFileNotValid if a .kmx is not valid in <Files>', async function() {
testForMessage(this, ['invalid', 'keyboardfilenotvalid.kps'], CompilerMessages.ERROR_KeyboardFileNotValid);
await testForMessage(this, ['invalid', 'keyboardfilenotvalid.kps'], CompilerMessages.ERROR_KeyboardFileNotValid);
});
// INFO_KeyboardFileHasNoKeyboardVersion
it('should generate INFO_KeyboardFileHasNoKeyboardVersion if <FollowKeyboardVersion> is set but keyboard has no version', async function() {
testForMessage(this, ['invalid', 'nokeyboardversion.kps'], CompilerMessages.INFO_KeyboardFileHasNoKeyboardVersion);
await testForMessage(this, ['invalid', 'nokeyboardversion.kps'], CompilerMessages.INFO_KeyboardFileHasNoKeyboardVersion);
});
// ERROR_PackageCannotContainBothModelsAndKeyboards
it('should generate ERROR_PackageCannotContainBothModelsAndKeyboards if package has both keyboards and models', async function() {
testForMessage(this, ['invalid', 'error_package_cannot_contain_both_models_and_keyboards.kps'], CompilerMessages.ERROR_PackageCannotContainBothModelsAndKeyboards);
await testForMessage(this, ['invalid', 'error_package_cannot_contain_both_models_and_keyboards.kps'], CompilerMessages.ERROR_PackageCannotContainBothModelsAndKeyboards);
});
// HINT_PackageShouldNotRepeatLanguages (models)
it('should generate HINT_PackageShouldNotRepeatLanguages if model has same language repeated', async function() {
testForMessage(this, ['invalid', 'keyman.en.hint_package_should_not_repeat_languages.model.kps'], CompilerMessages.HINT_PackageShouldNotRepeatLanguages);
await testForMessage(this, ['invalid', 'keyman.en.hint_package_should_not_repeat_languages.model.kps'], CompilerMessages.HINT_PackageShouldNotRepeatLanguages);
});
// HINT_PackageShouldNotRepeatLanguages (keyboards)
it('should generate HINT_PackageShouldNotRepeatLanguages if keyboard has same language repeated', async function() {
testForMessage(this, ['invalid', 'hint_package_should_not_repeat_languages.kps'], CompilerMessages.HINT_PackageShouldNotRepeatLanguages);
await testForMessage(this, ['invalid', 'hint_package_should_not_repeat_languages.kps'], CompilerMessages.HINT_PackageShouldNotRepeatLanguages);
});
// WARN_PackageNameDoesNotFollowLexicalModelConventions
it('should generate WARN_PackageNameDoesNotFollowLexicalModelConventions if filename has wrong conventions', async function() {
testForMessage(this, ['invalid', 'WARN_PackageNameDoesNotFollowLexicalModelConventions.kps'], CompilerMessages.WARN_PackageNameDoesNotFollowLexicalModelConventions);
await testForMessage(this, ['invalid', 'WARN_PackageNameDoesNotFollowLexicalModelConventions.kps'], CompilerMessages.WARN_PackageNameDoesNotFollowLexicalModelConventions);
});
// WARN_PackageNameDoesNotFollowKeyboardConventions
it('should generate WARN_PackageNameDoesNotFollowKeyboardConventions if filename has wrong conventions', async function() {
testForMessage(this, ['invalid', 'WARN_PackageNameDoesNotFollowKeyboardConventions.kps'], CompilerMessages.WARN_PackageNameDoesNotFollowKeyboardConventions);
await testForMessage(this, ['invalid', 'WARN_PackageNameDoesNotFollowKeyboardConventions.kps'], CompilerMessages.WARN_PackageNameDoesNotFollowKeyboardConventions);
});
// WARN_FileInPackageDoesNotFollowFilenameConventions
it('should generate WARN_FileInPackageDoesNotFollowFilenameConventions if content filename has wrong conventions', async function() {
testForMessage(this, ['invalid', 'warn_file_in_package_does_not_follow_filename_conventions.kps'],
await testForMessage(this, ['invalid', 'warn_file_in_package_does_not_follow_filename_conventions.kps'],
CompilerMessages.WARN_FileInPackageDoesNotFollowFilenameConventions, {checkFilenameConventions: true});
testForMessage(this, ['invalid', 'warn_file_in_package_does_not_follow_filename_conventions_2.kps'],
await testForMessage(this, ['invalid', 'warn_file_in_package_does_not_follow_filename_conventions_2.kps'],
CompilerMessages.WARN_FileInPackageDoesNotFollowFilenameConventions, {checkFilenameConventions: true});
});
// Test the inverse -- no warning generated if checkFilenameConventions is false
it('should not generate WARN_FileInPackageDoesNotFollowFilenameConventions if content filename has wrong conventions but checkFilenameConventions is false', async function() {
testForMessage(this, ['invalid', 'warn_file_in_package_does_not_follow_filename_conventions.kps'], null, {checkFilenameConventions: false});
testForMessage(this, ['invalid', 'warn_file_in_package_does_not_follow_filename_conventions_2.kps'], null, {checkFilenameConventions: false});
await testForMessage(this, ['invalid', 'warn_file_in_package_does_not_follow_filename_conventions.kps'], null, {checkFilenameConventions: false});
await testForMessage(this, ['invalid', 'warn_file_in_package_does_not_follow_filename_conventions_2.kps'], null, {checkFilenameConventions: false});
});
// ERROR_PackageNameCannotBeBlank
it('should generate ERROR_PackageNameCannotBeBlank if package info has empty name', async function() {
testForMessage(this, ['invalid', 'error_package_name_cannot_be_blank.kps'], CompilerMessages.ERROR_PackageNameCannotBeBlank); // blank field
testForMessage(this, ['invalid', 'error_package_name_cannot_be_blank_2.kps'], CompilerMessages.ERROR_PackageNameCannotBeBlank); // missing field
await testForMessage(this, ['invalid', 'error_package_name_cannot_be_blank.kps'], CompilerMessages.ERROR_PackageNameCannotBeBlank); // blank field
await testForMessage(this, ['invalid', 'error_package_name_cannot_be_blank_2.kps'], CompilerMessages.ERROR_PackageNameCannotBeBlank); // missing field
});
// ERROR_KeyboardFileNotFound
it('should generate ERROR_KeyboardFileNotFound if a <Keyboard> is listed in a package but not found in <Files>', async function() {
testForMessage(this, ['invalid', 'keyboardfilenotfound.kps'], CompilerMessages.ERROR_KeyboardFileNotFound);
await testForMessage(this, ['invalid', 'keyboardfilenotfound.kps'], CompilerMessages.ERROR_KeyboardFileNotFound);
});
// WARN_KeyboardVersionsDoNotMatch
it('should generate WARN_KeyboardVersionsDoNotMatch if two <Keyboards> have different versions', async function() {
testForMessage(this, ['invalid', 'warn_keyboard_versions_do_not_match.kps'], CompilerMessages.WARN_KeyboardVersionsDoNotMatch);
await testForMessage(this, ['invalid', 'warn_keyboard_versions_do_not_match.kps'], CompilerMessages.WARN_KeyboardVersionsDoNotMatch);
});
// ERROR_LanguageTagIsNotValid
@ -163,73 +154,73 @@ describe('CompilerMessages', function () {
// HINT_LanguageTagIsNotMinimal
it('should generate HINT_LanguageTagIsNotMinimal if keyboard has a non-minimal language tag', async function() {
testForMessage(this, ['invalid', 'hint_language_tag_is_not_minimal.kps'], CompilerMessages.HINT_LanguageTagIsNotMinimal);
await testForMessage(this, ['invalid', 'hint_language_tag_is_not_minimal.kps'], CompilerMessages.HINT_LanguageTagIsNotMinimal);
});
// ERROR_ModelMustHaveAtLeastOneLanguage
it('should generate ERROR_MustHaveAtLeastOneLanguage if model has zero language tags', async function() {
testForMessage(this, ['invalid', 'keyman.en.error_model_must_have_at_least_one_language.model.kps'],
await testForMessage(this, ['invalid', 'keyman.en.error_model_must_have_at_least_one_language.model.kps'],
CompilerMessages.ERROR_ModelMustHaveAtLeastOneLanguage);
});
// WARN_RedistFileShouldNotBeInPackage
it('should generate WARN_RedistFileShouldNotBeInPackage if package contains a redist file', async function() {
testForMessage(this, ['invalid', 'warn_redist_file_should_not_be_in_package.kps'],
await testForMessage(this, ['invalid', 'warn_redist_file_should_not_be_in_package.kps'],
CompilerMessages.WARN_RedistFileShouldNotBeInPackage);
});
// WARN_DocFileDangerous
it('should generate WARN_DocFileDangerous if package contains a .doc file', async function() {
testForMessage(this, ['invalid', 'warn_doc_file_dangerous.kps'],
await testForMessage(this, ['invalid', 'warn_doc_file_dangerous.kps'],
CompilerMessages.WARN_DocFileDangerous);
});
// ERROR_PackageMustContainAPackageOrAKeyboard
it('should generate ERROR_PackageMustContainAModelOrAKeyboard if package contains no keyboard or model', async function() {
testForMessage(this, ['invalid', 'error_package_must_contain_a_model_or_a_keyboard.kps'],
await testForMessage(this, ['invalid', 'error_package_must_contain_a_model_or_a_keyboard.kps'],
CompilerMessages.ERROR_PackageMustContainAModelOrAKeyboard);
});
// WARN_JsKeyboardFileIsMissing
it('should generate WARN_JsKeyboardFileIsMissing if package is missing corresponding .js for a touch .kmx', async function() {
testForMessage(this, ['invalid', 'warn_js_keyboard_file_is_missing.kps'],
await testForMessage(this, ['invalid', 'warn_js_keyboard_file_is_missing.kps'],
CompilerMessages.WARN_JsKeyboardFileIsMissing);
});
// WARN_KeyboardShouldHaveAtLeastOneLanguage
it('should generate WARN_KeyboardShouldHaveAtLeastOneLanguage if keyboard has zero language tags', async function() {
testForMessage(this, ['invalid', 'warn_keyboard_should_have_at_least_one_language.kps'],
await testForMessage(this, ['invalid', 'warn_keyboard_should_have_at_least_one_language.kps'],
CompilerMessages.WARN_KeyboardShouldHaveAtLeastOneLanguage);
});
// HINT_JsKeyboardFileHasNoTouchTargets
it('should generate HINT_JsKeyboardFileHasNoTouchTargets if keyboard has no touch targets', async function() {
testForMessage(this, ['invalid', 'hint_js_keyboard_file_has_no_touch_targets.kps'],
await testForMessage(this, ['invalid', 'hint_js_keyboard_file_has_no_touch_targets.kps'],
CompilerMessages.HINT_JsKeyboardFileHasNoTouchTargets);
});
it('should not generate HINT_JsKeyboardFileHasNoTouchTargets if keyboard has a touch target', async function() {
testForMessage(this, ['khmer_angkor', 'source', 'khmer_angkor.kps'], null);
await testForMessage(this, ['khmer_angkor', 'source', 'khmer_angkor.kps'], null);
});
// HINT_PackageContainsSourceFile
it('should generate HINT_PackageContainsSourceFile if package contains a source file', async function() {
testForMessage(this, ['invalid', 'hint_source_file_should_not_be_in_package.kps'],
await testForMessage(this, ['invalid', 'hint_source_file_should_not_be_in_package.kps'],
CompilerMessages.HINT_PackageContainsSourceFile);
});
// ERROR_InvalidPackageFile
it('should generate ERROR_InvalidPackageFile if package source file contains invalid XML', async function() {
testForMessage(this, ['invalid', 'error_invalid_package_file.kps'],
await testForMessage(this, ['invalid', 'error_invalid_package_file.kps'],
CompilerMessages.ERROR_InvalidPackageFile);
});

15
developer/src/kmc-package/test/test-package-compiler.ts
View file

@ -20,9 +20,13 @@ describe('KmpCompiler', function () {
'example.qaa.sencoten',
'withfolders.qaa.sencoten',
];
const callbacks = new TestCompilerCallbacks();
let kmpCompiler = new KmpCompiler(callbacks);
let kmpCompiler: KmpCompiler = null;
this.beforeAll(async function() {
kmpCompiler = new KmpCompiler();
assert.isTrue(await kmpCompiler.init(callbacks, null));
});
for (let modelID of MODELS) {
const kpsPath = modelID.includes('withfolders') ?
@ -89,7 +93,9 @@ describe('KmpCompiler', function () {
const kpsPath = makePathToFixture('khmer_angkor', 'source', 'khmer_angkor.kps');
const kmpJsonRefPath = makePathToFixture('khmer_angkor', 'ref', 'kmp.json');
const kmpCompiler = new KmpCompiler(callbacks);
const kmpCompiler = new KmpCompiler();
assert.isTrue(await kmpCompiler.init(callbacks, null));
const kmpJsonFixture: KmpJsonFile.KmpJsonFile = JSON.parse(fs.readFileSync(kmpJsonRefPath, 'utf-8'));
// We override the fixture version so that we can compare with the compiler output
@ -193,7 +199,8 @@ describe('KmpCompiler', function () {
callbacks.clear();
const kpsPath = makePathToFixture('absolute_path', 'source', 'absolute_path.kps');
const kmpCompiler = new KmpCompiler(callbacks);
const kmpCompiler = new KmpCompiler();
assert.isTrue(await kmpCompiler.init(callbacks, null));
let kmpJson: KmpJsonFile.KmpJsonFile = null;

4
developer/src/kmc-package/test/test-versioning.ts
View file

@ -33,7 +33,9 @@ describe('package versioning', function () {
for(const [ caseTitle, filename ] of cases) {
it(caseTitle, async function () {
const callbacks = new TestCompilerCallbacks();
const kmpCompiler = new KmpCompiler(callbacks);
const kmpCompiler = new KmpCompiler();
assert.isTrue(await kmpCompiler.init(callbacks, null));
const kpsPath = makePathToFixture('versioning', filename);
const kmpJson: KmpJsonFile.KmpJsonFile = kmpCompiler.transformKpsToKmpObject(kpsPath);
assert.isTrue(kmpJson !== null);

12
developer/src/kmc-package/test/test-windows-package-installer-compiler.ts
View file

@ -12,9 +12,6 @@ describe('WindowsPackageInstallerCompiler', function () {
it(`should build an SFX archive`, async function () {
this.timeout(10000); // this test can take a little while to run
const callbacks = new TestCompilerCallbacks();
let compiler = new WindowsPackageInstallerCompiler(callbacks);
const kpsPath = makePathToFixture('khmer_angkor', 'source', 'khmer_angkor.kps');
const sources: WindowsPackageInstallerSources = {
licenseFilename: makePathToFixture('windows-installer', 'license.txt'),
@ -25,12 +22,19 @@ describe('WindowsPackageInstallerCompiler', function () {
appName: 'Testing',
};
const sfxBuffer = await compiler.compile(kpsPath, sources);
const callbacks = new TestCompilerCallbacks();
let compiler = new WindowsPackageInstallerCompiler();
assert.isTrue(await compiler.init(callbacks, {sources}));
const result = await compiler.run(kpsPath, null);
assert.isNotNull(result);
// This returns a buffer with a SFX loader and a zip suffix. For the sake of repository size
// we actually provide a stub SFX loader and a stub MSI file, which is enough to verify that
// the compiler is generating what it thinks is a valid file.
const sfxBuffer = result.artifacts.exe.data;
const zip = JSZip();
// Check that file.kmp contains just 3 files - setup.inf, keymandesktop.msi, and khmer_angkor.kmp,

2
developer/src/kmc/src/commands/analyze.ts
View file

@ -105,7 +105,7 @@ async function analyzeOskCharUse(callbacks: CompilerCallbacks, filenames: string
async function analyzeOskRewritePua(callbacks: CompilerCallbacks, filenames: string[], options: AnalysisActivityOptions) {
const analyzer = new AnalyzeOskRewritePua(callbacks);
const mapping: any = JSON.parse(callbacks.fs.readFileSync(options.mappingFile, 'UTF-8'));
const mapping: any = JSON.parse(fs.readFileSync(options.mappingFile, 'utf-8'));
return await runOnFiles(callbacks, filenames, async (filename: string): Promise<boolean> => {
if(!await analyzer.analyze(filename, mapping)) {

15
developer/src/kmc/src/commands/build.ts
View file

@ -19,7 +19,6 @@ function commandOptionsToCompilerOptions(options: any): ExtendedCompilerOptions
// CompilerOptions properties...
return {
// CompilerBaseOptions
outFile: options.outFile,
logLevel: options.logLevel,
logFormat: options.logFormat,
color: options.color,
@ -66,7 +65,8 @@ File lists can be referenced with @filelist.txt.
If no input file is supplied, kmc will build the current folder.`)
.action(async (filenames: string[], _options: any, commander: any) => {
const options = commandOptionsToCompilerOptions(commander.optsWithGlobals());
const commanderOptions/*:{TODO?} CommandLineCompilerOptions*/ = commander.optsWithGlobals();
const options = commandOptionsToCompilerOptions(commanderOptions);
const callbacks = new NodeCompilerCallbacks(options);
if(!filenames.length) {
@ -75,12 +75,17 @@ If no input file is supplied, kmc will build the current folder.`)
filenames.push('.');
}
if(filenames.length > 1 && commanderOptions.outFile) {
// -o can only be specified with a single input file
callbacks.reportMessage(InfrastructureMessages.Error_OutFileCanOnlyBeSpecifiedWithSingleInfile());
}
if(!expandFileLists(filenames, callbacks)) {
process.exit(1);
}
for(let filename of filenames) {
if(!await build(filename, callbacks, options)) {
if(!await build(filename, commanderOptions.outFile, callbacks, options)) {
// Once a file fails to build, we bail on subsequent builds
process.exit(1);
}
@ -105,7 +110,7 @@ If no input file is supplied, kmc will build the current folder.`)
.action(buildWindowsPackageInstaller);
}
async function build(filename: string, parentCallbacks: NodeCompilerCallbacks, options: CompilerOptions): Promise<boolean> {
async function build(filename: string, outfile: string, parentCallbacks: NodeCompilerCallbacks, options: CompilerOptions): Promise<boolean> {
try {
// TEST: allow command-line simulation of infrastructure fatal errors, and
// also for unit tests
@ -151,7 +156,7 @@ async function build(filename: string, parentCallbacks: NodeCompilerCallbacks, o
const callbacks = new CompilerFileCallbacks(buildFilename, options, parentCallbacks);
callbacks.reportMessage(InfrastructureMessages.Info_BuildingFile({filename:buildFilename, relativeFilename}));
let result = await builder.build(filename, callbacks, options);
let result = await builder.build(filename, outfile, callbacks, options);
result = result && !callbacks.hasFailureMessage();
if(result) {
callbacks.reportMessage(builder instanceof BuildProject

41
developer/src/kmc/src/commands/buildClasses/BuildActivity.ts
View file

@ -1,15 +1,42 @@
import { CompilerCallbacks, CompilerOptions, KeymanFileTypes } from "@keymanapp/common-types";
import { escapeRegExp } from "../../util/escapeRegExp.js";
import * as fs from 'fs';
import * as path from 'path';
import { CompilerCallbacks, CompilerOptions, KeymanCompiler, KeymanFileTypes } from "@keymanapp/common-types";
import { InfrastructureMessages } from '../../messages/infrastructureMessages.js';
export abstract class BuildActivity {
public abstract get name(): string;
public abstract get sourceExtension(): KeymanFileTypes.Source;
public abstract get compiledExtension(): KeymanFileTypes.Binary;
public abstract get description(): string;
public abstract build(infile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise<boolean>;
protected getOutputFilename(infile: string, options: CompilerOptions): string {
return options.outFile ?
options.outFile :
infile.replace(new RegExp(escapeRegExp(this.sourceExtension), "g"), this.compiledExtension);
public abstract build(infile: string, outfile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise<boolean>;
protected async runCompiler<T extends CompilerOptions>(compiler: KeymanCompiler, infile: string, outfile: string, callbacks: CompilerCallbacks, options: T): Promise<boolean> {
if(!await compiler.init(callbacks, options)) {
return false;
}
const result = await compiler.run(infile, outfile);
if(!result) {
return false;
}
if(!this.createOutputFolder(outfile ?? infile, callbacks)) {
return false;
}
return await compiler.write(result.artifacts);
}
private createOutputFolder(targetFilename: string, callbacks: CompilerCallbacks): boolean {
const targetFolder = path.dirname(targetFilename);
try {
fs.mkdirSync(targetFolder, {recursive: true});
} catch(e) {
callbacks.reportMessage(InfrastructureMessages.Error_CannotCreateFolder({folderName:targetFolder, e}));
return false;
}
return true;
}
};

24
developer/src/kmc/src/commands/buildClasses/BuildKeyboardInfo.ts
View file

@ -13,7 +13,7 @@ export class BuildKeyboardInfo extends BuildActivity {
public get sourceExtension(): KeymanFileTypes.Source { return KeymanFileTypes.Source.Project; }
public get compiledExtension(): KeymanFileTypes.Binary { return KeymanFileTypes.Binary.KeyboardInfo; }
public get description(): string { return 'Build a keyboard metadata file'; }
public async build(infile: string, callbacks: CompilerCallbacks, options: ExtendedCompilerOptions): Promise<boolean> {
public async build(infile: string, _outfile: string, callbacks: CompilerCallbacks, options: ExtendedCompilerOptions): Promise<boolean> {
if(!KeymanFileTypes.filenameIs(infile, KeymanFileTypes.Source.Project)) {
// Even if the project file does not exist, we use its name as our reference
// in order to avoid ambiguity
@ -35,29 +35,17 @@ export class BuildKeyboardInfo extends BuildActivity {
const keyboard = project.files.find(file => file.fileType == KeymanFileTypes.Source.KeymanKeyboard);
const jsFilename = keyboard ? project.resolveOutputFilePath(keyboard, KeymanFileTypes.Source.KeymanKeyboard, KeymanFileTypes.Binary.WebKeyboard) : null;
const lastCommitDate = getLastGitCommitDate(project.projectPath);
const compiler = new KeyboardInfoCompiler(callbacks);
const data = await compiler.writeKeyboardInfoFile({
const sources = {
kmpFilename: project.resolveOutputFilePath(kps, KeymanFileTypes.Source.Package, KeymanFileTypes.Binary.Package),
kpsFilename: project.resolveInputFilePath(kps),
jsFilename: jsFilename && fs.existsSync(jsFilename) ? jsFilename : undefined,
sourcePath: calculateSourcePath(infile),
lastCommitDate,
forPublishing: !!options.forPublishing,
});
if(data == null) {
// Error messages have already been emitted by KeyboardInfoCompiler
return false;
}
};
// Note: should we always ignore the passed-in output filename for .keyboard_info?
const outputFilename = project.getOutputFilePath(KeymanFileTypes.Binary.KeyboardInfo);
fs.writeFileSync(
outputFilename,
data
);
return true;
const compiler = new KeyboardInfoCompiler();
return await super.runCompiler(compiler, infile, outputFilename, callbacks, {...options, sources});
}
}

26
developer/src/kmc/src/commands/buildClasses/BuildKmnKeyboard.ts
View file

@ -3,33 +3,21 @@ import { platform } from 'os';
import { KmnCompiler } from '@keymanapp/kmc-kmn';
import { CompilerOptions, CompilerCallbacks, KeymanFileTypes } from '@keymanapp/common-types';
import { BuildActivity } from './BuildActivity.js';
import * as fs from 'fs';
import { InfrastructureMessages } from '../../messages/infrastructureMessages.js';
export class BuildKmnKeyboard extends BuildActivity {
public get name(): string { return 'Keyman keyboard'; }
public get sourceExtension(): KeymanFileTypes.Source { return KeymanFileTypes.Source.KeymanKeyboard; }
public get compiledExtension(): KeymanFileTypes.Binary { return KeymanFileTypes.Binary.Keyboard; }
public get description(): string { return 'Build a Keyman keyboard'; }
public async build(infile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise<boolean> {
let compiler = new KmnCompiler();
if(!await compiler.init(callbacks)) {
return false;
public async build(infile: string, outfile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise<boolean> {
// We need to resolve paths to absolute paths before calling kmc-kmn
infile = getPosixAbsolutePath(infile);
if(outfile) {
outfile = getPosixAbsolutePath(outfile);
}
// We need to resolve paths to absolute paths before calling kmc-kmn
if(options.outFile) {
options.outFile = getPosixAbsolutePath(options.outFile);
const folderName = path.dirname(options.outFile);
try {
fs.mkdirSync(folderName, {recursive: true});
} catch(e) {
callbacks.reportMessage(InfrastructureMessages.Error_CannotCreateFolder({folderName, e}));
return false;
}
}
infile = getPosixAbsolutePath(infile);
return compiler.run(infile, options);
const compiler = new KmnCompiler();
return await super.runCompiler(compiler, infile, outfile, callbacks, options);
}
}

94
developer/src/kmc/src/commands/buildClasses/BuildLdmlKeyboard.ts
View file

@ -1,101 +1,19 @@
import * as path from 'path';
import * as fs from 'fs';
import * as kmcLdml from '@keymanapp/kmc-ldml';
import { KvkFileWriter, CompilerCallbacks, LDMLKeyboardXMLSourceFileReader, CompilerOptions, defaultCompilerOptions, KeymanFileTypes } from '@keymanapp/common-types';
import { CompilerCallbacks, LDMLKeyboardXMLSourceFileReader, CompilerOptions, KeymanFileTypes } from '@keymanapp/common-types';
import { BuildActivity } from './BuildActivity.js';
import { fileURLToPath } from 'url';
import { InfrastructureMessages } from '../../messages/infrastructureMessages.js';
export class BuildLdmlKeyboard extends BuildActivity {
public get name(): string { return 'LDML keyboard'; }
public get sourceExtension(): KeymanFileTypes.Source { return KeymanFileTypes.Source.LdmlKeyboard; }
public get compiledExtension(): KeymanFileTypes.Binary { return KeymanFileTypes.Binary.Keyboard; }
public get description(): string { return 'Build a LDML keyboard'; }
public async build(infile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise<boolean> {
public async build(infile: string, outfile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise<boolean> {
// TODO-LDML: consider hardware vs touch -- touch-only layout will not have a .kvk
// Compile:
let [kmx,kvk,kmw] = await buildLdmlKeyboardToMemory(infile, callbacks, options);
// Output:
const fileBaseName = options.outFile ?? infile;
const outFileBase = path.basename(fileBaseName, path.extname(fileBaseName));
const outFileDir = path.dirname(fileBaseName);
try {
fs.mkdirSync(outFileDir, {recursive: true});
} catch(e) {
callbacks.reportMessage(InfrastructureMessages.Error_CannotCreateFolder({folderName:outFileDir, e}));
return false;
}
if(kmx && kvk) {
const outFileKmx = path.join(outFileDir, outFileBase + KeymanFileTypes.Binary.Keyboard);
// TODO: console needs to be replaced with InfrastructureMessages
console.log(`Writing compiled keyboard to ${outFileKmx}`);
fs.writeFileSync(outFileKmx, kmx);
const outFileKvk = path.join(outFileDir, outFileBase + KeymanFileTypes.Binary.VisualKeyboard);
// TODO: console needs to be replaced with InfrastructureMessages
console.log(`Writing compiled visual keyboard to ${outFileKvk}`);
fs.writeFileSync(outFileKvk, kvk);
} else {
// TODO: console needs to be replaced with InfrastructureMessages
console.error(`An error occurred compiling ${infile}`);
return false;
}
if(kmw) {
const outFileKmw = path.join(outFileDir, outFileBase + KeymanFileTypes.Binary.WebKeyboard);
// TODO: console needs to be replaced with InfrastructureMessages
console.log(`Writing compiled js keyboard to ${outFileKmw}`);
fs.writeFileSync(outFileKmw, kmw);
}
return true;
}
}
async function buildLdmlKeyboardToMemory(inputFilename: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise<[Uint8Array, Uint8Array, Uint8Array]> {
let compilerOptions: kmcLdml.LdmlCompilerOptions = {
...defaultCompilerOptions,
...options,
readerOptions: {
const ldmlCompilerOptions: kmcLdml.LdmlCompilerOptions = {...options, readerOptions: {
importsPath: fileURLToPath(new URL(...LDMLKeyboardXMLSourceFileReader.defaultImportsURL))
}
};
const k = new kmcLdml.LdmlKeyboardCompiler(callbacks, compilerOptions);
let source = k.load(inputFilename);
if (!source) {
return [null, null, null];
}};
const compiler = new kmcLdml.LdmlKeyboardCompiler();
return await super.runCompiler(compiler, infile, outfile, callbacks, ldmlCompilerOptions);
}
let kmx = await k.compile(source);
if (!kmx) {
return [null, null, null];
}
// In order for the KMX file to be loaded by non-KMXPlus components, it is helpful
// to duplicate some of the metadata
kmcLdml.KMXPlusMetadataCompiler.addKmxMetadata(kmx.kmxplus, kmx.keyboard, compilerOptions);
// Use the builder to generate the binary output file
const builder = new kmcLdml.KMXBuilder(kmx, options.saveDebug);
const kmx_binary = builder.compile();
const vkcompiler = new kmcLdml.LdmlKeyboardVisualKeyboardCompiler(callbacks);
const vk = vkcompiler.compile(source);
const writer = new KvkFileWriter();
const kvk_binary = writer.write(vk);
// Note: we could have a step of generating source files here
// KvksFileWriter()...
// const tlcompiler = new kmc.TouchLayoutCompiler();
// const tl = tlcompiler.compile(source);
// const tlwriter = new TouchLayoutFileWriter();
const kmwcompiler = new kmcLdml.LdmlKeyboardKeymanWebCompiler(callbacks, compilerOptions);
const kmw_string = kmwcompiler.compile(inputFilename, source);
const encoder = new TextEncoder();
const kmw_binary = encoder.encode(kmw_string);
return [kmx_binary, kvk_binary, kmw_binary];
}

27
developer/src/kmc/src/commands/buildClasses/BuildModel.ts
View file

@ -1,6 +1,5 @@
import * as fs from 'fs';
import { BuildActivity } from './BuildActivity.js';
import { compileModel } from '@keymanapp/kmc-model';
import { LexicalModelCompiler } from '@keymanapp/kmc-model';
import { CompilerCallbacks, CompilerOptions, KeymanFileTypes } from '@keymanapp/common-types';
export class BuildModel extends BuildActivity {
@ -8,26 +7,8 @@ export class BuildModel extends BuildActivity {
public get sourceExtension(): KeymanFileTypes.Source { return KeymanFileTypes.Source.Model; }
public get compiledExtension(): KeymanFileTypes.Binary { return KeymanFileTypes.Binary.Model; }
public get description(): string { return 'Build a lexical model'; }
public async build(infile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise<boolean> {
let outputFilename: string = this.getOutputFilename(infile, options);
let code = null;
// Compile:
try {
code = compileModel(infile, callbacks);
} catch(e) {
console.error(e);
return false;
}
if(!code) {
console.error('Compilation failed.')
return false;
}
// Output:
fs.writeFileSync(outputFilename, code, 'utf8');
return true;
public async build(infile: string, outfile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise<boolean> {
const compiler = new LexicalModelCompiler();
return await super.runCompiler(compiler, infile, outfile, callbacks, options);
}
}

32
developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts
View file

@ -1,4 +1,4 @@
import * as fs from 'fs';
import * as path from 'path';
import { BuildActivity } from './BuildActivity.js';
import { CompilerCallbacks, KeymanFileTypes } from '@keymanapp/common-types';
import { ModelInfoCompiler } from '@keymanapp/kmc-model-info';
@ -25,7 +25,7 @@ export class BuildModelInfo extends BuildActivity {
* @param options
* @returns
*/
public async build(infile: string, callbacks: CompilerCallbacks, options: ExtendedCompilerOptions): Promise<boolean> {
public async build(infile: string, _outfile: string, callbacks: CompilerCallbacks, options: ExtendedCompilerOptions): Promise<boolean> {
if(!KeymanFileTypes.filenameIs(infile, KeymanFileTypes.Source.Project)) {
// Even if the project file does not exist, we use its name as our reference
// in order to avoid ambiguity
@ -50,7 +50,12 @@ export class BuildModelInfo extends BuildActivity {
return false;
}
let kmpCompiler = new KmpCompiler(callbacks);
let kmpCompiler = new KmpCompiler();
if(!await kmpCompiler.init(callbacks, options)) {
// Errors will have been emitted by KmpCompiler
return false;
}
let kmpJsonData = kmpCompiler.transformKpsToKmpObject(project.resolveInputFilePath(kps));
if(!kmpJsonData) {
// Errors will have been emitted by KmpCompiler
@ -58,9 +63,8 @@ export class BuildModelInfo extends BuildActivity {
}
const lastCommitDate = getLastGitCommitDate(project.projectPath);
const compiler = new ModelInfoCompiler(callbacks);
const data = compiler.writeModelMetadataFile({
model_id: callbacks.path.basename(project.projectPath, KeymanFileTypes.Source.Project),
const sources = {
model_id: path.basename(project.projectPath, KeymanFileTypes.Source.Project),
kmpJsonData,
sourcePath: calculateSourcePath(infile),
modelFileName: project.resolveOutputFilePath(model, KeymanFileTypes.Source.Model, KeymanFileTypes.Binary.Model),
@ -68,18 +72,12 @@ export class BuildModelInfo extends BuildActivity {
kpsFilename: project.resolveInputFilePath(kps),
lastCommitDate,
forPublishing: !!options.forPublishing,
});
};
if(data == null) {
// Error messages have already been emitted by writeModelMetadataFile
return false;
}
// Note: should we always ignore the passed-in output filename for .model_info?
const outputFilename = project.getOutputFilePath(KeymanFileTypes.Binary.ModelInfo);
fs.writeFileSync(
project.getOutputFilePath(KeymanFileTypes.Binary.ModelInfo),
data
);
return true;
const compiler = new ModelInfoCompiler();
return await super.runCompiler(compiler, infile, outputFilename, callbacks, {...options, sources});
}
}

40
developer/src/kmc/src/commands/buildClasses/BuildPackage.ts
View file

@ -1,46 +1,14 @@
import * as fs from 'fs';
import { BuildActivity } from './BuildActivity.js';
import { CompilerCallbacks, CompilerOptions, KeymanFileTypes } from '@keymanapp/common-types';
import { KmpCompiler, PackageValidation } from '@keymanapp/kmc-package';
import { KmpCompiler } from '@keymanapp/kmc-package';
export class BuildPackage extends BuildActivity {
public get name(): string { return 'Package'; }
public get sourceExtension(): KeymanFileTypes.Source { return KeymanFileTypes.Source.Package; }
public get compiledExtension(): KeymanFileTypes.Binary { return KeymanFileTypes.Binary.Package; }
public get description(): string { return 'Build a Keyman package'; }
public async build(infile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise<boolean> {
const outfile = this.getOutputFilename(infile, options);
//
// Load .kps source data
//
const kmpCompiler = new KmpCompiler(callbacks);
const kmpJsonData = kmpCompiler.transformKpsToKmpObject(infile);
if(!kmpJsonData) {
return false;
}
//
// Validate the package file
//
const validation = new PackageValidation(callbacks, options);
if(!validation.validate(infile, kmpJsonData)) {
return false;
}
//
// Build the .kmp package file
//
const data = await kmpCompiler.buildKmpFile(infile, kmpJsonData);
if(!data) {
return false;
}
fs.writeFileSync(outfile, data, 'binary');
return true;
public async build(infile: string, outfile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise<boolean> {
const compiler = new KmpCompiler();
return await super.runCompiler(compiler, infile, outfile, callbacks, options);
}
}

18
developer/src/kmc/src/commands/buildClasses/BuildProject.ts
View file

@ -12,7 +12,12 @@ export class BuildProject extends BuildActivity {
public get sourceExtension(): KeymanFileTypes.Source { return KeymanFileTypes.Source.Project; }
public get compiledExtension(): KeymanFileTypes.Binary { return null; }
public get description(): string { return 'Build a keyboard or lexical model project'; }
public async build(infile: string, callbacks: CompilerCallbacks, options: ExtendedCompilerOptions): Promise<boolean> {
public async build(infile: string, outfile: string, callbacks: CompilerCallbacks, options: ExtendedCompilerOptions): Promise<boolean> {
if(outfile) {
callbacks.reportMessage(InfrastructureMessages.Error_OutFileNotValidForProjects());
return false;
}
let builder = new ProjectBuilder(infile, callbacks, options);
return builder.run();
}
@ -31,11 +36,6 @@ class ProjectBuilder {
}
async run(): Promise<boolean> {
if(this.options.outFile) {
this.callbacks.reportMessage(InfrastructureMessages.Error_OutFileNotValidForProjects());
return false;
}
this.project = loadProject(this.infile, this.callbacks);
if(!this.project) {
return false;
@ -102,7 +102,7 @@ class ProjectBuilder {
async buildTarget(file: KeymanDeveloperProjectFile, activity: BuildActivity): Promise<boolean> {
const options = {...this.options};
options.outFile = this.project.resolveOutputFilePath(file, activity.sourceExtension, activity.compiledExtension);
const outfile = this.project.resolveOutputFilePath(file, activity.sourceExtension, activity.compiledExtension);
options.checkFilenameConventions = this.project.options.checkFilenameConventions ?? this.options.checkFilenameConventions;
const infile = this.project.resolveInputFilePath(file);
@ -110,9 +110,9 @@ class ProjectBuilder {
const callbacks = new CompilerFileCallbacks(buildFilename, options, this.callbacks);
callbacks.reportMessage(InfrastructureMessages.Info_BuildingFile({filename: infile, relativeFilename:buildFilename}));
fs.mkdirSync(path.dirname(options.outFile), {recursive:true});
fs.mkdirSync(path.dirname(outfile), {recursive:true});
let result = await activity.build(infile, callbacks, options);
let result = await activity.build(infile, outfile, callbacks, options);
// check if we had a message that causes the build to be a failure
// note: command line option here, if set, overrides project setting

18
developer/src/kmc/src/commands/buildTestData/index.ts
View file

@ -1,12 +1,13 @@
import * as fs from 'fs';
import * as path from 'path';
import * as kmcLdml from '@keymanapp/kmc-ldml';
import { CompilerBaseOptions, CompilerCallbacks, defaultCompilerOptions, LDMLKeyboardTestDataXMLSourceFile, LDMLKeyboardXMLSourceFileReader } from '@keymanapp/common-types';
import { CompilerCallbacks, defaultCompilerOptions, LDMLKeyboardTestDataXMLSourceFile, LDMLKeyboardXMLSourceFileReader } from '@keymanapp/common-types';
import { NodeCompilerCallbacks } from '../../util/NodeCompilerCallbacks.js';
import { fileURLToPath } from 'url';
import { CommandLineBaseOptions } from 'src/util/baseOptions.js';
export function buildTestData(infile: string, _options: any, commander: any) {
const options: CompilerBaseOptions = commander.optsWithGlobals();
export async function buildTestData(infile: string, _options: any, commander: any) {
const options: CommandLineBaseOptions = commander.optsWithGlobals();
let compilerOptions: kmcLdml.LdmlCompilerOptions = {
...defaultCompilerOptions,
@ -18,7 +19,7 @@ export function buildTestData(infile: string, _options: any, commander: any) {
}
};
let testData = loadTestData(infile, compilerOptions);
let testData = await loadTestData(infile, compilerOptions);
if (!testData) {
return;
}
@ -31,12 +32,11 @@ export function buildTestData(infile: string, _options: any, commander: any) {
fs.writeFileSync(outFileJson, JSON.stringify(testData, null, ' '));
}
function loadTestData(inputFilename: string, options: kmcLdml.LdmlCompilerOptions): LDMLKeyboardTestDataXMLSourceFile {
async function loadTestData(inputFilename: string, options: kmcLdml.LdmlCompilerOptions): Promise<LDMLKeyboardTestDataXMLSourceFile> {
const callbacks: CompilerCallbacks = new NodeCompilerCallbacks(options);
const k = new kmcLdml.LdmlKeyboardCompiler(callbacks, options);
let source = k.loadTestData(inputFilename);
if (!source) {
const k = new kmcLdml.LdmlKeyboardCompiler();
if(!await k.init(callbacks, options)) {
return null;
}
return source;
return await k.loadTestData(inputFilename);
}

33
developer/src/kmc/src/commands/buildWindowsPackageInstaller/index.ts
View file

@ -1,10 +1,11 @@
import * as fs from 'fs';
import * as path from 'path';
import { CompilerBaseOptions, CompilerCallbacks, defaultCompilerOptions } from '@keymanapp/common-types';
import { CompilerCallbacks, defaultCompilerOptions } from '@keymanapp/common-types';
import { NodeCompilerCallbacks } from '../../util/NodeCompilerCallbacks.js';
import { WindowsPackageInstallerCompiler, WindowsPackageInstallerSources } from '@keymanapp/kmc-package';
import { CommandLineBaseOptions } from 'src/util/baseOptions.js';
interface WindowsPackageInstallerOptions extends CompilerBaseOptions {
interface WindowsPackageInstallerCommandLineOptions extends CommandLineBaseOptions {
msi: string;
exe: string;
license: string;
@ -15,7 +16,9 @@ interface WindowsPackageInstallerOptions extends CompilerBaseOptions {
};
export async function buildWindowsPackageInstaller(infile: string, _options: any, commander: any) {
const options: WindowsPackageInstallerOptions = commander.optsWithGlobals();
// TODO(lowpri): we probably should cleanup the options management here, move
// translation of command line options to kmc-* options into a separate module
const options: WindowsPackageInstallerCommandLineOptions = commander.optsWithGlobals();
const sources: WindowsPackageInstallerSources = {
licenseFilename: options.license,
msiFilename: options.msi,
@ -26,22 +29,30 @@ export async function buildWindowsPackageInstaller(infile: string, _options: any
titleImageFilename: options.titleImage
}
const outfile: string = options.outFile;
// Normalize case for the filename and expand the path; this avoids false
// positive case mismatches on input filenames and glommed paths
infile = fs.realpathSync.native(infile);
const callbacks: CompilerCallbacks = new NodeCompilerCallbacks({...defaultCompilerOptions, ...options});
const compiler = new WindowsPackageInstallerCompiler(callbacks);
const compiler = new WindowsPackageInstallerCompiler();
if(!await compiler.init(callbacks, {...options, sources})) {
process.exit(1);
}
const buffer = await compiler.compile(infile, sources);
if(!buffer) {
const fileBaseName = outfile ?? infile;
const outFileBase = path.basename(fileBaseName, path.extname(fileBaseName));
const outFileDir = path.dirname(fileBaseName);
const outFileExe = path.join(outFileDir, outFileBase + '.exe');
const result = await compiler.run(infile, outFileExe);
if(!result) {
// errors will have been reported already
process.exit(1);
}
const fileBaseName = options.outFile ?? infile;
const outFileBase = path.basename(fileBaseName, path.extname(fileBaseName));
const outFileDir = path.dirname(fileBaseName);
const outFileExe = path.join(outFileDir, outFileBase + '.exe');
fs.writeFileSync(outFileExe, buffer);
if(!await compiler.write(result.artifacts)) {
process.exit(1);
}
}

18
developer/src/kmc/src/kmlmc.ts
View file

@ -3,9 +3,8 @@
* kmlmc - Keyman Lexical Model Compiler
*/
import * as fs from 'fs';
import { Command } from 'commander';
import { compileModel } from '@keymanapp/kmc-model';
import { LexicalModelCompiler } from '@keymanapp/kmc-model';
import { SysExits } from './util/sysexits.js';
import KEYMAN_VERSION from "@keymanapp/keyman-version";
import { NodeCompilerCallbacks } from './util/NodeCompilerCallbacks.js';
@ -30,10 +29,16 @@ if (!inputFilename) {
const callbacks = new NodeCompilerCallbacks({logLevel: 'info'});
const compiler = new LexicalModelCompiler();
if(!await compiler.init(callbacks, null)) {
console.error('Initialization failed.');
process.exit(SysExits.EX_DATAERR);
}
let code = null;
// Compile:
try {
code = compileModel(inputFilename, callbacks);
code = await compiler.run(inputFilename, program.opts().outFile);
} catch(e) {
console.error(e);
process.exit(SysExits.EX_DATAERR);
@ -46,9 +51,12 @@ if(!code) {
// Output:
if (program.opts().outFile) {
fs.writeFileSync(program.opts().outFile, code, 'utf8');
compiler.write(code.artifacts);
} else {
console.log(code);
// TODO(lowpri): if writing to console then log messages should all be to stderr?
const decoder = new TextDecoder();
const text = decoder.decode(code.artifacts.js.data);
console.log(text);
}
function exitDueToUsageError(message: string): never {

31
developer/src/kmc/src/kmlmp.ts
View file

@ -5,9 +5,8 @@
// Note: this is a deprecated package and will be removed in Keyman 18.0
import * as fs from 'fs';
import { Command } from 'commander';
import { PackageValidation, KmpCompiler } from '@keymanapp/kmc-package';
import { KmpCompiler } from '@keymanapp/kmc-package';
import { SysExits } from './util/sysexits.js';
import KEYMAN_VERSION from "@keymanapp/keyman-version";
import { NodeCompilerCallbacks } from './util/NodeCompilerCallbacks.js';
@ -34,36 +33,24 @@ if (!inputFilename) {
let outputFilename: string = program.opts().outFile ? program.opts().outFile : inputFilename.replace(/\.kps$/, ".kmp");
//
// Load .kps source data
// Run the compiler
//
const callbacks = new NodeCompilerCallbacks({logLevel: 'info'});
let kmpCompiler = new KmpCompiler(callbacks);
let kmpJsonData = kmpCompiler.transformKpsToKmpObject(inputFilename);
if(!kmpJsonData) {
let kmpCompiler = new KmpCompiler();
if(!await kmpCompiler.init(callbacks, null)) {
process.exit(1);
}
//
// Validate the package file
//
const validation = new PackageValidation(callbacks, {});
if(!validation.validate(inputFilename, kmpJsonData)) {
let result = await kmpCompiler.run(inputFilename, outputFilename);
if(!result) {
process.exit(1);
}
//
// Build the .kmp package file
//
let promise = kmpCompiler.buildKmpFile(inputFilename, kmpJsonData);
promise.then(data => {
fs.writeFileSync(outputFilename, data, 'binary');
}).catch(error => {
// Consumer decides how to handle errors
if(!await kmpCompiler.write(result.artifacts)) {
console.error('Failed to write kmp file');
});
process.exit(1);
}
function exitDueToUsageError(message: string): never {
console.error(`${program.name()}: ${message}`);

4
developer/src/kmc/src/messages/infrastructureMessages.ts
View file

@ -91,5 +91,9 @@ export class InfrastructureMessages {
static Hint_ProjectIsVersion10 = () => m(this.HINT_ProjectIsVersion10,
`The project file is an older version and can be upgraded to version 17.0`);
static HINT_ProjectIsVersion10 = SevHint | 0x0014;
static Error_OutFileCanOnlyBeSpecifiedWithSingleInfile = () => m(this.ERROR_OutFileCanOnlyBeSpecifiedWithSingleInfile,
`Parameter --out-file can only be used with a single input file.`);
static ERROR_OutFileCanOnlyBeSpecifiedWithSingleInfile = SevError | 0x0015;
}

2
developer/src/kmc/src/util/TestKeymanSentry.ts
View file

@ -26,7 +26,7 @@ export class TestKeymanSentry {
if(cli.includes('kmcmplib')) {
const compiler = new KmnCompiler();
const callbacks = new NodeCompilerCallbacks({});
if(!await compiler.init(callbacks)) {
if(!await compiler.init(callbacks, null)) {
throw new Error('Failed to instantiate WASM compiler');
}
try {

19
developer/src/kmc/src/util/baseOptions.ts
View file

@ -1,8 +1,23 @@
import { ALL_COMPILER_LOG_FORMATS, ALL_COMPILER_LOG_LEVELS } from "@keymanapp/common-types";
import { ALL_COMPILER_LOG_FORMATS, ALL_COMPILER_LOG_LEVELS, CompilerLogFormat, CompilerLogLevel } from "@keymanapp/common-types";
import { Command, Option } from "commander";
// These options map to CompilerBaseOptions
/**
* Abstract interface for compiler options
*/
export interface CommandLineBaseOptions {
// These options map to CompilerBaseOptions
logLevel?: CompilerLogLevel;
logFormat?: CompilerLogFormat;
color?: boolean;
// This option is not in CompilerBaseOptions
outFile?:string;
}
/**
* These options map to CompilerBaseOptions
*/
export class BaseOptions {
public static addLogLevel(program: Command) {
return program.addOption(new Option('-l, --log-level <logLevel>', 'Log level').choices(ALL_COMPILER_LOG_LEVELS).default('info'));

7
developer/src/kmc/src/util/escapeRegExp.ts
View file

@ -1,7 +0,0 @@
const escapedRegexp = /[.*+?^${}()|[\]\\]/g;
// See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions#escaping
export function escapeRegExp(s: string) {
return s.replace(escapedRegexp, "\\$&"); // $& means the whole matched string
}

2
developer/src/kmc/test/test-build.ts
View file

@ -34,7 +34,7 @@ describe('compilerWarningsAsErrors', function () {
const builder = new BuildProject();
const path = makePathToFixture('compiler-warnings-as-errors',
`compiler_warnings_as_errors_${truth.kpj === true ? 'true' : (truth.kpj === false ? 'false' : 'undefined')}.kpj`);
const result = await builder.build(path, callbacks, {
const result = await builder.build(path, null, callbacks, {
compilerWarningsAsErrors: truth.cli,
});
if(truth.result != result) {

6
developer/src/kmc/test/test-infrastructureMessages.ts
View file

@ -24,7 +24,7 @@ describe('InfrastructureMessages', function () {
const expectedMessages = [InfrastructureMessages.FATAL_UnexpectedException];
process.env.SENTRY_CLIENT_TEST_BUILD_EXCEPTION = '1';
await unitTestEndpoints.build(null, ncb, {});
await unitTestEndpoints.build(null, null, ncb, {});
delete process.env.SENTRY_CLIENT_TEST_BUILD_EXCEPTION;
assertMessagesEqual(ncb.messages, expectedMessages);
@ -79,7 +79,7 @@ describe('InfrastructureMessages', function () {
InfrastructureMessages.INFO_WarningsHaveFailedBuild,
InfrastructureMessages.INFO_FileNotBuiltSuccessfully
];
await unitTestEndpoints.build(filename, ncb, {compilerWarningsAsErrors: true});
await unitTestEndpoints.build(filename, null, ncb, {compilerWarningsAsErrors: true});
assertMessagesEqual(ncb.messages, expectedMessages);
});
@ -93,7 +93,7 @@ describe('InfrastructureMessages', function () {
InfrastructureMessages.ERROR_UnsupportedProjectVersion,
InfrastructureMessages.INFO_ProjectNotBuiltSuccessfully
];
await unitTestEndpoints.build(filename, ncb, {compilerWarningsAsErrors: true});
await unitTestEndpoints.build(filename, null, ncb, {compilerWarningsAsErrors: true});
assertMessagesEqual(ncb.messages, expectedMessages);
});
});

2
developer/src/kmc/test/test-project-build.ts
View file

@ -11,7 +11,7 @@ describe('BuildProject', function () {
it('should build a keyboard project', async function() {
const builder = new BuildProject();
const path = makePathToFixture('relative_paths', 'k_000___null_keyboard.kpj');
let result = await builder.build(path, callbacks, {
let result = await builder.build(path, null, callbacks, {
shouldAddCompilerVersion: false,
compilerWarningsAsErrors: true,
saveDebug: false,