From 192dafa6ea4f584aae235bf65894dbabbbadab4e Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 17 Jul 2023 08:15:45 +0700 Subject: [PATCH] feat(developer): support file lists in kmc Fixes #9162. Adds support for `@files.txt` parameters in kmc. Files listed in this file, separated by line breaks, will be added to the input file list, relative to the path of files.txt. The file can include comments in lines starting with '#' and ignores whitespace before and after each filename. --- developer/src/kmc/src/commands/build.ts | 20 +++++- developer/src/kmc/src/util/fileLists.ts | 49 +++++++++++++++ .../kmc/test/fixtures/file-lists/files.txt | 15 +++++ developer/src/kmc/test/test-fileLists.ts | 61 +++++++++++++++++++ 4 files changed, 143 insertions(+), 2 deletions(-) create mode 100644 developer/src/kmc/src/util/fileLists.ts create mode 100644 developer/src/kmc/test/fixtures/file-lists/files.txt create mode 100644 developer/src/kmc/test/test-fileLists.ts diff --git a/developer/src/kmc/src/commands/build.ts b/developer/src/kmc/src/commands/build.ts index bca00b74b9..60e91ddcfb 100644 --- a/developer/src/kmc/src/commands/build.ts +++ b/developer/src/kmc/src/commands/build.ts @@ -7,6 +7,7 @@ import { NodeCompilerCallbacks } from '../util/NodeCompilerCallbacks.js'; import { InfrastructureMessages } from '../messages/messages.js'; import { CompilerFileCallbacks, CompilerOptions, KeymanFileTypes } from '@keymanapp/common-types'; import { BaseOptions } from '../util/baseOptions.js'; +import { expandFileLists } from '../util/fileLists.js'; function commandOptionsToCompilerOptions(options: any): CompilerOptions { @@ -29,7 +30,19 @@ function commandOptionsToCompilerOptions(options: any): CompilerOptions { export function declareBuild(program: Command) { BaseOptions.addAll(program .command('build [infile...]') - .description('Build a source file into a final file') + .description(`Compile one or more source files or projects.`) + .addHelpText('after', ` +Supported file types: + * folder: Keyman project in folder + * .kpj: Keyman project + * .kmn: Keyman keyboard + * .xml: LDML keyboard + * .model.ts: Keyman lexical model + * .kps: Keyman keyboard package + +File lists can be referenced with @filelist.txt. + +If no input file is supplied, kmc will build the current folder.`) ) .option('-d, --debug', 'Include debug information in output') .option('-w, --compiler-warnings-as-errors', 'Causes warnings to fail the build; overrides project-level warnings-as-errors option') @@ -48,10 +61,13 @@ export function declareBuild(program: Command) { filenames.push('.'); } + if(!expandFileLists(filenames, callbacks)) { + process.exit(1); + } + for(let filename of filenames) { if(!await build(filename, callbacks, options)) { // Once a file fails to build, we bail on subsequent builds - // TODO: is this the most appropriate semantics? process.exit(1); } } diff --git a/developer/src/kmc/src/util/fileLists.ts b/developer/src/kmc/src/util/fileLists.ts new file mode 100644 index 0000000000..11ce21905e --- /dev/null +++ b/developer/src/kmc/src/util/fileLists.ts @@ -0,0 +1,49 @@ +import * as fs from 'fs'; +import * as path from 'path'; +import { CompilerCallbacks } from "@keymanapp/common-types"; +import { InfrastructureMessages } from "../messages/messages.js"; + +/** + * Replaces each entry starting with `@` with the content of the file, with one + * line per file, filenames trimmed, and any lines that are blank or starting + * with `#` (marking a comment) removed. Note: `#` anywhere else is treated as + * part of the filename. + * + * If any filelist does not exist, reports an error and returns false. + * + * @param filenames + * @param callbacks + * @returns false on failure + */ +export function expandFileLists(filenames: string[], callbacks: CompilerCallbacks) { + let i = 0; + while(i < filenames.length) { + if(filenames[i].startsWith('@')) { + const fileList = expandFileList(filenames[i].substring(1), callbacks); + if(fileList === null) { + return false; + } + filenames.splice(i, 1, ...fileList); + i += fileList.length; + } else { + i++; + } + } + return true; +} + +export function expandFileList(filename: string, callbacks: CompilerCallbacks): string[] { + if(!fs.existsSync(filename)) { + callbacks.reportMessage(InfrastructureMessages.Error_FileDoesNotExist({filename})); + return null; + } + + const files = fs.readFileSync(filename, 'utf-8').split('\n').map(item => { + item = item.trim(); + return item.startsWith('#') || item == '' + ? '' + : path.resolve(path.dirname(filename), item) + }).filter(item => item.length > 0); + + return files; +} diff --git a/developer/src/kmc/test/fixtures/file-lists/files.txt b/developer/src/kmc/test/fixtures/file-lists/files.txt new file mode 100644 index 0000000000..927f69c55e --- /dev/null +++ b/developer/src/kmc/test/fixtures/file-lists/files.txt @@ -0,0 +1,15 @@ + +# +# This is a comment +# + +file1.kmn + + # + # we want to compile these files too + # + + file2.kmn + file with space.kps + +# Some more commentary diff --git a/developer/src/kmc/test/test-fileLists.ts b/developer/src/kmc/test/test-fileLists.ts new file mode 100644 index 0000000000..5092ff849d --- /dev/null +++ b/developer/src/kmc/test/test-fileLists.ts @@ -0,0 +1,61 @@ +import { TestCompilerCallbacks } from '@keymanapp/developer-test-helpers'; +import { assert } from 'chai'; +import 'mocha'; +import { makePathToFixture } from './helpers/index.js'; +import { expandFileList, expandFileLists } from '../src/util/fileLists.js'; + +const callbacks = new TestCompilerCallbacks(); + +// expandFileList expands each file name relative to the file list's supplied +// filename, so we need to compare against the full path. +const expectedFileList = [ + makePathToFixture('file-lists', 'file1.kmn'), + makePathToFixture('file-lists', 'file2.kmn'), + makePathToFixture('file-lists', 'file with space.kps') +]; + +const expectedFiles = [ + 'file0.kmn', + ...expectedFileList, + 'file4.kmn', + 'file5.kmn' +]; + +beforeEach(function() { + callbacks.clear(); +}); + +describe('expandFileList', function () { + it('should report a missing filelist correctly', async function() { + const path = makePathToFixture('file-lists', 'does-not-exist.txt'); + + const fileList = expandFileList(path, callbacks); + assert.isNull(fileList); + assert.equal(callbacks.messages.length, 1); + }); + + it('should expand a list of files correctly', async function() { + const path = makePathToFixture('file-lists', 'files.txt'); + + const fileList = expandFileList(path, callbacks); + assert.isNotNull(fileList); + assert.equal(callbacks.messages.length, 0); + assert.deepEqual(fileList, expectedFileList); + }); +}); + +describe('expandFileLists', function () { + it('should splice a filelist in correctly', async function() { + // We just use this to test the splicing so no path resolution is made + const files = [ + 'file0.kmn', + '@' + makePathToFixture('file-lists', 'files.txt'), + 'file4.kmn', + 'file5.kmn' + ]; + + assert.isTrue(expandFileLists(files, callbacks)); + assert.equal(callbacks.messages.length, 0); + assert.deepEqual(files, expectedFiles); + }); +});