feat(developer): support file lists in kmc

Fixes #9162.

Adds support for `@files.txt` parameters in kmc. Files listed in this
file, separated by line breaks, will be added to the input file list,
relative to the path of files.txt.

The file can include comments in lines starting with '#' and ignores
whitespace before and after each filename.

developer/src/kmc/src/commands/build.ts

@@ -7,6 +7,7 @@ import { NodeCompilerCallbacks } from '../util/NodeCompilerCallbacks.js';
 import { InfrastructureMessages } from '../messages/messages.js';
 import { CompilerFileCallbacks, CompilerOptions, KeymanFileTypes } from '@keymanapp/common-types';
 import { BaseOptions } from '../util/baseOptions.js';
+import { expandFileLists } from '../util/fileLists.js';
 
 
 function commandOptionsToCompilerOptions(options: any): CompilerOptions {
@@ -29,7 +30,19 @@ function commandOptionsToCompilerOptions(options: any): CompilerOptions {
 export function declareBuild(program: Command) {
   BaseOptions.addAll(program
     .command('build [infile...]')
-    .description('Build a source file into a final file')
+    .description(`Compile one or more source files or projects.`)
+    .addHelpText('after', `
+Supported file types:
+  * folder: Keyman project in folder
+  * .kpj: Keyman project
+  * .kmn: Keyman keyboard
+  * .xml: LDML keyboard
+  * .model.ts: Keyman lexical model
+  * .kps: Keyman keyboard package
+
+File lists can be referenced with @filelist.txt.
+
+If no input file is supplied, kmc will build the current folder.`)
   )
     .option('-d, --debug', 'Include debug information in output')
     .option('-w, --compiler-warnings-as-errors', 'Causes warnings to fail the build; overrides project-level warnings-as-errors option')
@@ -48,10 +61,13 @@ export function declareBuild(program: Command) {
         filenames.push('.');
       }
 
+      if(!expandFileLists(filenames, callbacks)) {
+        process.exit(1);
+      }
+
       for(let filename of filenames) {
         if(!await build(filename, callbacks, options)) {
           // Once a file fails to build, we bail on subsequent builds
-          // TODO: is this the most appropriate semantics?
           process.exit(1);
         }
       }

developer/src/kmc/src/util/fileLists.ts

@@ -0,0 +1,49 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import { CompilerCallbacks } from "@keymanapp/common-types";
+import { InfrastructureMessages } from "../messages/messages.js";
+
+/**
+ * Replaces each entry starting with `@` with the content of the file, with one
+ * line per file, filenames trimmed, and any lines that are blank or starting
+ * with `#` (marking a comment) removed. Note: `#` anywhere else is treated as
+ * part of the filename.
+ *
+ * If any filelist does not exist, reports an error and returns false.
+ *
+ * @param filenames
+ * @param callbacks
+ * @returns false on failure
+ */
+export function expandFileLists(filenames: string[], callbacks: CompilerCallbacks) {
+  let i = 0;
+  while(i < filenames.length) {
+    if(filenames[i].startsWith('@')) {
+      const fileList = expandFileList(filenames[i].substring(1), callbacks);
+      if(fileList === null) {
+        return false;
+      }
+      filenames.splice(i, 1, ...fileList);
+      i += fileList.length;
+    } else {
+      i++;
+    }
+  }
+  return true;
+}
+
+export function expandFileList(filename: string, callbacks: CompilerCallbacks): string[] {
+  if(!fs.existsSync(filename)) {
+    callbacks.reportMessage(InfrastructureMessages.Error_FileDoesNotExist({filename}));
+    return null;
+  }
+
+  const files = fs.readFileSync(filename, 'utf-8').split('\n').map(item => {
+    item = item.trim();
+    return item.startsWith('#') || item == ''
+      ? ''
+      : path.resolve(path.dirname(filename), item)
+  }).filter(item => item.length > 0);
+
+  return files;
+}

developer/src/kmc/test/fixtures/file-lists/files.txt

@@ -0,0 +1,15 @@
+
+#
+# This is a comment
+#
+
+file1.kmn
+
+  #
+  # we want to compile these files too
+  #
+
+  file2.kmn
+  file with space.kps
+
+# Some more commentary

developer/src/kmc/test/test-fileLists.ts

@@ -0,0 +1,61 @@
+import { TestCompilerCallbacks } from '@keymanapp/developer-test-helpers';
+import { assert } from 'chai';
+import 'mocha';
+import { makePathToFixture } from './helpers/index.js';
+import { expandFileList, expandFileLists } from '../src/util/fileLists.js';
+
+const callbacks = new TestCompilerCallbacks();
+
+// expandFileList expands each file name relative to the file list's supplied
+// filename, so we need to compare against the full path.
+const expectedFileList = [
+  makePathToFixture('file-lists', 'file1.kmn'),
+  makePathToFixture('file-lists', 'file2.kmn'),
+  makePathToFixture('file-lists', 'file with space.kps')
+];
+
+const expectedFiles = [
+  'file0.kmn',
+  ...expectedFileList,
+  'file4.kmn',
+  'file5.kmn'
+];
+
+beforeEach(function() {
+  callbacks.clear();
+});
+
+describe('expandFileList', function () {
+  it('should report a missing filelist correctly', async function() {
+    const path = makePathToFixture('file-lists', 'does-not-exist.txt');
+
+    const fileList = expandFileList(path, callbacks);
+    assert.isNull(fileList);
+    assert.equal(callbacks.messages.length, 1);
+  });
+
+  it('should expand a list of files correctly', async function() {
+    const path = makePathToFixture('file-lists', 'files.txt');
+
+    const fileList = expandFileList(path, callbacks);
+    assert.isNotNull(fileList);
+    assert.equal(callbacks.messages.length, 0);
+    assert.deepEqual(fileList, expectedFileList);
+  });
+});
+
+describe('expandFileLists', function () {
+  it('should splice a filelist in correctly', async function() {
+    // We just use this to test the splicing so no path resolution is made
+    const files = [
+      'file0.kmn',
+      '@' + makePathToFixture('file-lists', 'files.txt'),
+      'file4.kmn',
+      'file5.kmn'
+    ];
+
+    assert.isTrue(expandFileLists(files, callbacks));
+    assert.equal(callbacks.messages.length, 0);
+    assert.deepEqual(files, expectedFiles);
+  });
+});