From a30fdffd954a7213e56a45f836bc00684f25a576 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Thu, 23 Feb 2023 13:16:01 +0700 Subject: [PATCH] feat(developer): kmc build command --- core/tests/unit/ldml/keyboards/meson.build | 4 +- developer/src/kmc/README.md | 2 +- developer/src/kmc/build.sh | 5 +- .../kmc/src/activities/buildLdmlKeyboard.ts | 81 ++++++++ .../src/kmc/src/activities/buildTestData.ts | 34 ++++ developer/src/kmc/src/commands/build.ts | 53 +++++ .../src/kmc/src/commands/buildTestData.ts | 14 ++ developer/src/kmc/src/kmc.ts | 190 +++--------------- .../src/kmc/src/util/NodeCompilerCallbacks.ts | 36 ++++ 9 files changed, 255 insertions(+), 164 deletions(-) create mode 100644 developer/src/kmc/src/activities/buildLdmlKeyboard.ts create mode 100644 developer/src/kmc/src/activities/buildTestData.ts create mode 100644 developer/src/kmc/src/commands/build.ts create mode 100644 developer/src/kmc/src/commands/buildTestData.ts create mode 100644 developer/src/kmc/src/util/NodeCompilerCallbacks.ts diff --git a/core/tests/unit/ldml/keyboards/meson.build b/core/tests/unit/ldml/keyboards/meson.build index 3c1418bf89..c04cd9a1a5 100644 --- a/core/tests/unit/ldml/keyboards/meson.build +++ b/core/tests/unit/ldml/keyboards/meson.build @@ -42,7 +42,7 @@ foreach kbd : tests ) configure_file( - command: kmc_cmd + ['@INPUT@', '--out-file', '@OUTPUT@'], + command: kmc_cmd + ['build', '@INPUT@', '--out-file', '@OUTPUT@'], output: kbd + '.kmx', input: kbd + '.xml' ) @@ -50,7 +50,7 @@ endforeach foreach kbd : tests_with_testdata configure_file( - command: kmc_cmd + ['@INPUT@', '--test-data', '@OUTPUT@'], + command: kmc_cmd + ['build-test-data', '@INPUT@', '@OUTPUT@'], output: kbd + '-test.json', input: kbd + '-test.xml' ) diff --git a/developer/src/kmc/README.md b/developer/src/kmc/README.md index 585eed7d56..f0c816bbb5 100644 --- a/developer/src/kmc/README.md +++ b/developer/src/kmc/README.md @@ -34,7 +34,7 @@ kmc Usage To compile an LDML keyboard from its `.xml` source, use `kmc`: - kmc my-keyboard.xml --outFile my-keyboard.kmx + kmc build my-keyboard.xml --outFile my-keyboard.kmx To see more command line options by using the `--help` option: diff --git a/developer/src/kmc/build.sh b/developer/src/kmc/build.sh index d774a311dd..56ee574f18 100755 --- a/developer/src/kmc/build.sh +++ b/developer/src/kmc/build.sh @@ -45,8 +45,9 @@ if builder_start_action clean; then else # We need the schema file at runtime and bundled, so always copy it for all actions except `clean` mkdir -p "$THIS_SCRIPT_PATH/build/src/" - cp "$KEYMAN_ROOT/resources/standards-data/ldml-keyboards/techpreview/ldml-keyboard.schema.json" "$THIS_SCRIPT_PATH/build/src/" - cp "$KEYMAN_ROOT/resources/standards-data/ldml-keyboards/techpreview/ldml-keyboardtest.schema.json" "$THIS_SCRIPT_PATH/build/src/" + cp "$KEYMAN_ROOT/resources/standards-data/ldml-keyboards/techpreview/ldml-keyboard.schema.json" "$THIS_SCRIPT_PATH/build/src/util/" + cp "$KEYMAN_ROOT/resources/standards-data/ldml-keyboards/techpreview/ldml-keyboardtest.schema.json" "$THIS_SCRIPT_PATH/build/src/util/" + cp "$KEYMAN_ROOT/common/schemas/kvks/kvks.schema.json" "$THIS_SCRIPT_PATH/build/src/util/" fi diff --git a/developer/src/kmc/src/activities/buildLdmlKeyboard.ts b/developer/src/kmc/src/activities/buildLdmlKeyboard.ts new file mode 100644 index 0000000000..54f0b768f8 --- /dev/null +++ b/developer/src/kmc/src/activities/buildLdmlKeyboard.ts @@ -0,0 +1,81 @@ +import * as path from 'path'; +import * as fs from 'fs'; +import * as kmc from '@keymanapp/kmc-keyboard'; +import { KvkFileWriter, CompilerCallbacks } from '@keymanapp/common-types'; +import { NodeCompilerCallbacks } from '../util/NodeCompilerCallbacks.js'; + +export function buildLdmlKeyboard(infile: string, options: any) { + // TODO-LDML: consider hardware vs touch -- touch-only layout will not have a .kvk + // Compile: + let [kmx,kvk,kmw] = buildLdmlKeyboardToMemory(infile, options); + // Output: + + const fileBaseName = options.outFile ?? infile; + const outFileBase = path.basename(fileBaseName, path.extname(fileBaseName)); + const outFileDir = path.dirname(fileBaseName); + + if(kmx && kvk) { + const outFileKmx = path.join(outFileDir, outFileBase + '.kmx'); + console.log(`Writing compiled keyboard to ${outFileKmx}`); + fs.writeFileSync(outFileKmx, kmx); + + const outFileKvk = path.join(outFileDir, outFileBase + '.kvk'); + console.log(`Writing compiled visual keyboard to ${outFileKvk}`); + fs.writeFileSync(outFileKvk, kvk); + } else { + console.error(`An error occurred compiling ${infile}`); + process.exit(1); + } + + if(kmw) { + const outFileKmw = path.join(outFileDir, outFileBase + '.js'); + console.log(`Writing compiled js keyboard to ${outFileKmw}`); + fs.writeFileSync(outFileKmw, kmw); + } +} + +function buildLdmlKeyboardToMemory(inputFilename: string, options: any): [Uint8Array, Uint8Array, Uint8Array] { + let compilerOptions: kmc.CompilerOptions = { + debug: options.debug ?? false, + addCompilerVersion: options.compilerVersion ?? true + } + + const c: CompilerCallbacks = new NodeCompilerCallbacks(); + const k = new kmc.Compiler(c, options); + let source = k.load(inputFilename); + if (!source) { + return [null, null, null]; + } + if (!k.validate(source)) { + return [null, null, null]; + } + let kmx = k.compile(source); + if (!kmx) { + return [null, null, null]; + } + + // In order for the KMX file to be loaded by non-KMXPlus components, it is helpful + // to duplicate some of the metadata + kmc.KMXPlusMetadataCompiler.addKmxMetadata(kmx.kmxplus, kmx.keyboard, compilerOptions); + + // Use the builder to generate the binary output file + const builder = new kmc.KMXBuilder(kmx, options.debug); + const kmx_binary = builder.compile(); + + const vkcompiler = new kmc.VisualKeyboardCompiler(); + const vk = vkcompiler.compile(source); + const writer = new KvkFileWriter(); + const kvk_binary = writer.write(vk); + + // Note: we could have a step of generating source files here + // KvksFileWriter()... + // const tlcompiler = new kmc.TouchLayoutCompiler(); + // const tl = tlcompiler.compile(source); + // const tlwriter = new TouchLayoutFileWriter(); + const kmwcompiler = new kmc.KeymanWebCompiler(compilerOptions); + const kmw_string = kmwcompiler.compile(inputFilename, source); + const encoder = new TextEncoder(); + const kmw_binary = encoder.encode(kmw_string); + + return [kmx_binary, kvk_binary, kmw_binary]; +} diff --git a/developer/src/kmc/src/activities/buildTestData.ts b/developer/src/kmc/src/activities/buildTestData.ts new file mode 100644 index 0000000000..0e8f424245 --- /dev/null +++ b/developer/src/kmc/src/activities/buildTestData.ts @@ -0,0 +1,34 @@ +import * as fs from 'fs'; +import * as path from 'path'; +import * as kmc from '@keymanapp/kmc-keyboard'; +import { CompilerCallbacks, LDMLKeyboardTestDataXMLSourceFile } from '@keymanapp/common-types'; +import { NodeCompilerCallbacks } from '../util/NodeCompilerCallbacks.js'; + +export function buildTestData(infile: string) { + let compilerOptions: kmc.CompilerOptions = { + debug: false, + addCompilerVersion: false + }; + + let testData = loadTestData(infile, compilerOptions); + if (!testData) { + return; + } + + const fileBaseName = infile; + const outFileBase = path.basename(fileBaseName, path.extname(fileBaseName)); + const outFileDir = path.dirname(fileBaseName); + const outFileJson = path.join(outFileDir, outFileBase + '.json'); + console.log(`Writing JSON test data to ${outFileJson}`); + fs + .writeFileSync(outFileJson, JSON.stringify(testData, null, ' ')); +} +function loadTestData(inputFilename: string, options: kmc.CompilerOptions): LDMLKeyboardTestDataXMLSourceFile { + const c: CompilerCallbacks = new NodeCompilerCallbacks(); + const k = new kmc.Compiler(c, options); + let source = k.loadTestData(inputFilename); + if (!source) { + return null; + } + return source; +} diff --git a/developer/src/kmc/src/commands/build.ts b/developer/src/kmc/src/commands/build.ts new file mode 100644 index 0000000000..8695201580 --- /dev/null +++ b/developer/src/kmc/src/commands/build.ts @@ -0,0 +1,53 @@ +import { Command } from 'commander'; +import { buildLdmlKeyboard } from '../activities/buildLdmlKeyboard.js'; + +export function declareBuild(program: Command) { + program + .command('build [infile...]') + .description('Build a source file into a final file') + .option('-d, --debug', 'Include debug information in output') + .option('-o, --out-file ', 'where to save the resulting .kmx file') + .option('--no-compiler-version', 'Exclude compiler version metadata from output') + .action((infiles: string[], options: any) => { + if(!infiles.length) { + console.debug('Assuming infile == .'); + build('.', options); + } + for(let infile of infiles) { + build(infile, options); + } + }); +} + +function build(infile: string, options: any) { + console.log(`Building ${infile}`); + + if(infile.endsWith('.xml')) { + return buildLdmlKeyboard(infile, options); + } + +/* + if(infile.endsWith('.kmn')) { + return buildKmnKeyboard(infile, options); + } + + if(infile.endsWith('.kps')) { + return buildPackage(infile, options); + } + + if(infile.endsWith('.model.ts')) { + return buildModel(infile, options); + } + + if(infile.endsWith('.kpj')) { + return buildProject(infile, options); + } + + if(fs.statSync(infile).isDirectory()) { + return buildProjectFolder(infile, options); + } +*/ + + console.error(`Unrecognised input file ${infile}, expecting .xml, .kmn, .kps, .model.ts, .kpj, or project folder`); + process.exit(2); +} diff --git a/developer/src/kmc/src/commands/buildTestData.ts b/developer/src/kmc/src/commands/buildTestData.ts new file mode 100644 index 0000000000..86f7b2cfe4 --- /dev/null +++ b/developer/src/kmc/src/commands/buildTestData.ts @@ -0,0 +1,14 @@ + + +import { Command } from 'commander'; +import { buildTestData } from '../activities/buildTestData.js'; + +export function declareBuildTestData(program: Command) { + program + .command('build-test-data ') + .description('Convert keyboard test .xml to .json') + .option('-o, --out-file ', 'where to save the resulting .kmx file') + .action(buildTestData); +} + + diff --git a/developer/src/kmc/src/kmc.ts b/developer/src/kmc/src/kmc.ts index 5e74a81f31..ddd677532a 100644 --- a/developer/src/kmc/src/kmc.ts +++ b/developer/src/kmc/src/kmc.ts @@ -3,172 +3,44 @@ * kmc - Keyman Next Generation Compiler */ -import * as fs from 'fs'; -import * as path from 'path'; import { Command } from 'commander'; -import * as kmc from '@keymanapp/kmc-keyboard'; import KEYMAN_VERSION from "@keymanapp/keyman-version/keyman-version.mjs"; -import { KvkFileWriter, CompilerCallbacks, CompilerEvent, LDMLKeyboardTestDataXMLSourceFile } from '@keymanapp/common-types'; - -let inputFilename: string; +import { declareBuild } from './commands/build.js'; +import { declareBuildTestData } from './commands/buildTestData.js'; const program = new Command(); + /* Arguments */ program - .description('Compiles Keyman LDML keyboards') - .version(KEYMAN_VERSION.VERSION_WITH_TAG) - .arguments('') - .action((infile:any) => inputFilename = infile) - .option('-d, --debug', 'Include debug information in output') - .option('--no-compiler-version', 'Exclude compiler version metadata from output') - .option('-o, --out-file ', 'where to save the resulting .kmx file') - .option('-T, --test-data ', 'Convert keyboard test .xml to .json'); + .description('Keyman Developer Command Line Interface') + .version(KEYMAN_VERSION.VERSION_WITH_TAG); + +declareBuild(program); +declareBuildTestData(program); + +/* +program + .command('clean'); + +program + .command('copy'); + +program + .command('rename'); + +program + .command('generate'); + +program + .command('import'); + +program + .command('test'); + +program + .command('publish'); +*/ program.parse(process.argv); -// Deal with input arguments: -if (!inputFilename) { - exitDueToUsageError('Must provide a LDML keyboard .xml source file.'); -} - -function exitDueToUsageError(message: string): never { - console.error(`${program._name}: ${message}`); - console.error(); - program.outputHelp(); - return process.exit(64); // TODO: SysExits.EX_USAGE -} - -/** - * Concrete implementation for CLI use - */ -class NodeCompilerCallbacks implements CompilerCallbacks { - loadFile(baseFilename: string, filename: string | URL): Buffer { - // TODO: translate filename based on the baseFilename - try { - return fs.readFileSync(filename); - } catch(e) { - if (e.code === 'ENOENT') { - return null; - } else { - throw e; - } - } - } - reportMessage(event: CompilerEvent): void { - console.log(kmc.CompilerMessages.severityName(event.code) + ' ' + event.code.toString(16) + ': ' + event.message); - } - loadLdmlKeyboardSchema(): Buffer { - let schemaPath = new URL('ldml-keyboard.schema.json', import.meta.url); - return fs.readFileSync(schemaPath); - } - loadLdmlKeyboardTestSchema(): Buffer { - let schemaPath = new URL('ldml-keyboardtest.schema.json', import.meta.url); - return fs.readFileSync(schemaPath); - } - loadKvksJsonSchema(): Buffer { - let schemaPath = new URL('kvks.schema.json', import.meta.url); - return fs.readFileSync(schemaPath); - } -} - -function compileKeyboard(inputFilename: string, options: kmc.CompilerOptions): [Uint8Array,Uint8Array,Uint8Array] { - const c : CompilerCallbacks = new NodeCompilerCallbacks(); - const k = new kmc.Compiler(c, options); - let source = k.load(inputFilename); - if(!source) { - return [null, null, null]; - } - if(!k.validate(source)) { - return [null, null, null]; - } - let kmx = k.compile(source); - if(!kmx) { - return [null, null, null]; - } - - // In order for the KMX file to be loaded by non-KMXPlus components, it is helpful - // to duplicate some of the metadata - kmc.KMXPlusMetadataCompiler.addKmxMetadata(kmx.kmxplus, kmx.keyboard, options); - - // Use the builder to generate the binary output file - const builder = new kmc.KMXBuilder(kmx, options.debug); - const kmx_binary = builder.compile(); - - const vkcompiler = new kmc.VisualKeyboardCompiler(); - const vk = vkcompiler.compile(source); - const writer = new KvkFileWriter(); - const kvk_binary = writer.write(vk); - - // Note: we could have a step of generating source files here - // KvksFileWriter()... - // const tlcompiler = new kmc.TouchLayoutCompiler(); - // const tl = tlcompiler.compile(source); - // const tlwriter = new TouchLayoutFileWriter(); - - const kmwcompiler = new kmc.KeymanWebCompiler(options); - const kmw_string = kmwcompiler.compile(inputFilename, source); - const encoder = new TextEncoder(); - const kmw_binary = encoder.encode(kmw_string); - - return [kmx_binary, kvk_binary, kmw_binary]; -} - -function loadTestData(inputFilename: string, options: kmc.CompilerOptions): LDMLKeyboardTestDataXMLSourceFile { - const c : CompilerCallbacks = new NodeCompilerCallbacks(); - const k = new kmc.Compiler(c, options); - let source = k.loadTestData(inputFilename); - if(!source) { - return null; - } - return source; -} - -let options: kmc.CompilerOptions = { - debug: program.debug ?? false, - addCompilerVersion: program.compilerVersion ?? true -} - -if (program.testData) { - let testData = loadTestData(inputFilename, options); - if (testData) { - const fileBaseName = program.testData ?? inputFilename; - const outFileBase = path.basename(fileBaseName, path.extname(fileBaseName)); - const outFileDir = path.dirname(fileBaseName); - - const outFileJson = path.join(outFileDir, outFileBase + '.json'); - console.log(`Writing JSON test data to ${outFileJson}`); - fs.writeFileSync(outFileJson, JSON.stringify(testData, null, ' ')); - } else { - console.error(`An error occurred loading test data ${inputFilename}`); - process.exit(1); - } -} else { - // TODO-LDML: consider hardware vs touch -- touch-only layout will not have a .kvk - // Compile: - let [kmx,kvk,kmw] = compileKeyboard(inputFilename, options); - // Output: - - const fileBaseName = program.outFile ?? inputFilename; - const outFileBase = path.basename(fileBaseName, path.extname(fileBaseName)); - const outFileDir = path.dirname(fileBaseName); - - if(kmx && kvk) { - const outFileKmx = path.join(outFileDir, outFileBase + '.kmx'); - console.log(`Writing compiled keyboard to ${outFileKmx}`); - fs.writeFileSync(outFileKmx, kmx); - - const outFileKvk = path.join(outFileDir, outFileBase + '.kvk'); - console.log(`Writing compiled visual keyboard to ${outFileKvk}`); - fs.writeFileSync(outFileKvk, kvk); - } else { - console.error(`An error occurred compiling ${inputFilename}`); - process.exit(1); - } - - if(kmw) { - const outFileKmw = path.join(outFileDir, outFileBase + '.js'); - console.log(`Writing compiled js keyboard to ${outFileKmw}`); - fs.writeFileSync(outFileKmw, kmw); - } -} diff --git a/developer/src/kmc/src/util/NodeCompilerCallbacks.ts b/developer/src/kmc/src/util/NodeCompilerCallbacks.ts new file mode 100644 index 0000000000..a64b919a56 --- /dev/null +++ b/developer/src/kmc/src/util/NodeCompilerCallbacks.ts @@ -0,0 +1,36 @@ +import * as fs from 'fs'; +import * as kmc from '@keymanapp/kmc-keyboard'; +import { CompilerCallbacks, CompilerEvent } from '@keymanapp/common-types'; + +/** + * Concrete implementation for CLI use + */ +export class NodeCompilerCallbacks implements CompilerCallbacks { + loadFile(baseFilename: string, filename: string | URL): Buffer { + // TODO: translate filename based on the baseFilename + try { + return fs.readFileSync(filename); + } catch (e) { + if (e.code === 'ENOENT') { + return null; + } else { + throw e; + } + } + } + reportMessage(event: CompilerEvent): void { + console.log(kmc.CompilerMessages.severityName(event.code) + ' ' + event.code.toString(16) + ': ' + event.message); + } + loadLdmlKeyboardSchema(): Buffer { + let schemaPath = new URL('ldml-keyboard.schema.json', import.meta.url); + return fs.readFileSync(schemaPath); + } + loadLdmlKeyboardTestSchema(): Buffer { + let schemaPath = new URL('ldml-keyboardtest.schema.json', import.meta.url); + return fs.readFileSync(schemaPath); + } + loadKvksJsonSchema(): Buffer { + let schemaPath = new URL('kvks.schema.json', import.meta.url); + return fs.readFileSync(schemaPath); + } +}