feat(developer): deploy kmc as ES module

Deploy kmc as an ES module, forced now by top-level await that we use in
the sentry load.

Moved NodeCompilerCallbacks to same util folder as schemas are found in,
to avoid rewriting schema loading with imports in this PR. It belongs
better there than messages/ anyway.

Distributed files are now .mjs. Removed --enable-source-maps from
launchers as logging will be managed with sentry anyway.

developer/src/inst/node/kmc.cmd

@@ -1,4 +1,4 @@
 @rem This script avoids path dependencies for node for distribution
 @rem with Keyman Developer. When used on platforms other than Windows,
 @rem node can be used directly with the compiler (`npm link` will setup).
-@"%~dp0\node.js\node.exe" --enable-source-maps "%~dp0\kmc\kmc.cjs" %*
+@"%~dp0\node.js\node.exe" "%~dp0\kmc\kmc.mjs" %*

developer/src/inst/node/kmlmc.cmd

@@ -1,4 +1,4 @@
 @rem This script avoids path dependencies for node for distribution
 @rem with Keyman Developer. When used on platforms other than Windows,
 @rem node can be used directly with the compiler (`npm link` will setup).
-@"%~dp0\node.js\node.exe" --enable-source-maps "%~dp0\kmc\kmlmc.cjs" %*
+@"%~dp0\node.js\node.exe" "%~dp0\kmc\kmlmc.mjs" %*

developer/src/inst/node/kmlmi.cmd

@@ -1,4 +1,4 @@
 @rem This script avoids path dependencies for node for distribution
 @rem with Keyman Developer. When used on platforms other than Windows,
 @rem node can be used directly with the compiler (`npm link` will setup).
-@"%~dp0\node.js\node.exe" --enable-source-maps "%~dp0\kmc\kmlmi.cjs" %*
+@"%~dp0\node.js\node.exe" "%~dp0\kmc\kmlmi.mjs" %*

developer/src/inst/node/kmlmp.cmd

@@ -1,4 +1,4 @@
 @rem This script avoids path dependencies for node for distribution
 @rem with Keyman Developer. When used on platforms other than Windows,
 @rem node can be used directly with the compiler (`npm link` will setup).
-@"%~dp0\node.js\node.exe" --enable-source-maps "%~dp0\kmc\kmlmp.cjs" %*
+@"%~dp0\node.js\node.exe" "%~dp0\kmc\kmlmp.mjs" %*

developer/src/kmc-model/build.sh

@@ -32,7 +32,6 @@ builder_parse "$@"
 #-------------------------------------------------------------------------------------------------------------------
 
 function do_build() {
-  mkdir -p build/cjs-src
   npm run build
 }
 

developer/src/kmc/build-bundler.js

@@ -0,0 +1,32 @@
+/*
+ * Bundle kmc as esm with appropriate node modules and banner
+ */
+
+import esbuild from 'esbuild';
+
+await esbuild.build({
+  entryPoints: [
+    'build/src/kmc.js',
+    'build/src/kmlmc.js',
+    'build/src/kmlmi.js',
+    'build/src/kmlmp.js',
+  ],
+  bundle: true,
+  format: 'esm',
+  platform: 'node',
+  target: 'es2022',
+  outdir: 'build/dist/',
+
+  // We want a .mjs extension to force node into ESM module mode
+  outExtension: { '.js': '.mjs' },
+
+  // Thunk for external modules such as Commander that are still CJS and still
+  // use require, __filename, __dirname
+  banner: {
+    js: `
+      const require = (await import("node:module")).createRequire(import.meta.url);
+      const __filename = (await import("node:url")).fileURLToPath(import.meta.url);
+      const __dirname = (await import("node:path")).dirname(__filename);
+    `,
+  },
+});

developer/src/kmc/build.sh

@@ -56,13 +56,20 @@ fi
 
 function copy_schemas() {
   # We need the schema file at runtime and bundled, so always copy it for all actions except `clean`
+  local schemas=(
+    "$KEYMAN_ROOT/resources/standards-data/ldml-keyboards/techpreview/ldml-keyboard.schema.json"
+    "$KEYMAN_ROOT/resources/standards-data/ldml-keyboards/techpreview/ldml-keyboardtest.schema.json"
+    "$KEYMAN_ROOT/common/schemas/kvks/kvks.schema.json"
+    "$KEYMAN_ROOT/common/schemas/kpj/kpj.schema.json"
+    "$KEYMAN_ROOT/common/schemas/kpj-9.0/kpj-9.0.schema.json"
+    "$KEYMAN_ROOT/common/schemas/displaymap/displaymap.schema.json"
+  )
+
   mkdir -p "$THIS_SCRIPT_PATH/build/src/util/"
-  cp "$KEYMAN_ROOT/resources/standards-data/ldml-keyboards/techpreview/ldml-keyboard.schema.json" "$THIS_SCRIPT_PATH/build/src/util/"
-  cp "$KEYMAN_ROOT/resources/standards-data/ldml-keyboards/techpreview/ldml-keyboardtest.schema.json" "$THIS_SCRIPT_PATH/build/src/util/"
-  cp "$KEYMAN_ROOT/common/schemas/kvks/kvks.schema.json" "$THIS_SCRIPT_PATH/build/src/util/"
-  cp "$KEYMAN_ROOT/common/schemas/kpj/kpj.schema.json" "$THIS_SCRIPT_PATH/build/src/util/"
-  cp "$KEYMAN_ROOT/common/schemas/kpj-9.0/kpj-9.0.schema.json" "$THIS_SCRIPT_PATH/build/src/util/"
-  cp "$KEYMAN_ROOT/common/schemas/displaymap/displaymap.schema.json" "$THIS_SCRIPT_PATH/build/src/util/"
+  cp "${schemas[@]}" "$THIS_SCRIPT_PATH/build/src/util/"
+
+  mkdir -p "$THIS_SCRIPT_PATH/build/dist/"
+  cp "${schemas[@]}" "$THIS_SCRIPT_PATH/build/dist/"
 }
 
 #-------------------------------------------------------------------------------------------------------------------
@@ -94,16 +101,21 @@ fi
 #-------------------------------------------------------------------------------------------------------------------
 
 if builder_start_action bundle; then
-  copy_schemas
 
   if ! builder_has_option --build-path; then
     builder_finish_action "Parameter --build-path is required" bundle
     exit 64
   fi
 
-  mkdir -p build/cjs-src
-  npm run bundle
-  cp build/cjs-src/* "$BUILD_PATH"
+  rm -rf build/dist
+  mkdir -p build/dist
+  node build-bundler.js
+
+  # Manually copy over kmcmplib module and schemas
+  copy_schemas
+  cp ../kmc-kmn/build/src/import/kmcmplib/wasm-host.wasm build/dist/
+
+  cp build/dist/* "$BUILD_PATH"
 
   builder_finish_action success bundle
 fi

developer/src/kmc/src/commands/analyze.ts

@@ -1,7 +1,7 @@
 import * as fs from 'fs';
 import * as path from 'path';
 import { Command, Option } from 'commander';
-import { NodeCompilerCallbacks } from '../messages/NodeCompilerCallbacks.js';
+import { NodeCompilerCallbacks } from '../util/NodeCompilerCallbacks.js';
 import { InfrastructureMessages } from '../messages/messages.js';
 import { CompilerCallbacks, CompilerLogLevel } from '@keymanapp/common-types';
 import { AnalyzeOskCharacterUse, AnalyzeOskRewritePua } from '@keymanapp/kmc-analyze';

developer/src/kmc/src/commands/build.ts

@@ -2,7 +2,7 @@ import * as fs from 'fs';
 import { Command } from 'commander';
 import { buildActivities } from './buildClasses/buildActivities.js';
 import { BuildProject } from './buildClasses/BuildProject.js';
-import { NodeCompilerCallbacks } from '../messages/NodeCompilerCallbacks.js';
+import { NodeCompilerCallbacks } from '../util/NodeCompilerCallbacks.js';
 import { InfrastructureMessages } from '../messages/messages.js';
 import { CompilerFileCallbacks, CompilerOptions, KeymanFileTypes } from '@keymanapp/common-types';
 import { BaseOptions } from '../util/baseOptions.js';

developer/src/kmc/src/commands/buildTestData/index.ts

@@ -2,7 +2,7 @@ import * as fs from 'fs';
 import * as path from 'path';
 import * as kmcLdml from '@keymanapp/kmc-ldml';
 import { CompilerBaseOptions, CompilerCallbacks, defaultCompilerOptions, LDMLKeyboardTestDataXMLSourceFile, LDMLKeyboardXMLSourceFileReader } from '@keymanapp/common-types';
-import { NodeCompilerCallbacks } from '../../messages/NodeCompilerCallbacks.js';
+import { NodeCompilerCallbacks } from '../../util/NodeCompilerCallbacks.js';
 import { fileURLToPath } from 'url';
 
 export function buildTestData(infile: string, options: CompilerBaseOptions) {

developer/src/kmc/src/kmlmc.ts

@@ -8,7 +8,7 @@ import { Command } from 'commander';
 import { compileModel } from '@keymanapp/kmc-model';
 import { SysExits } from './util/sysexits.js';
 import KEYMAN_VERSION from "@keymanapp/keyman-version";
-import { NodeCompilerCallbacks } from './messages/NodeCompilerCallbacks.js';
+import { NodeCompilerCallbacks } from './util/NodeCompilerCallbacks.js';
 
 let inputFilename: string;
 const program = new Command();

developer/src/kmc/src/kmlmi.ts

@@ -9,7 +9,7 @@ import { KmpCompiler, PackageValidation } from '@keymanapp/kmc-package';
 import { ModelInfoOptions, writeMergedModelMetadataFile } from '@keymanapp/kmc-model-info';
 import { SysExits } from './util/sysexits.js';
 import KEYMAN_VERSION from "@keymanapp/keyman-version";
-import { NodeCompilerCallbacks } from './messages/NodeCompilerCallbacks.js';
+import { NodeCompilerCallbacks } from './util/NodeCompilerCallbacks.js';
 
 let inputFilename: string;
 const program = new Command();

developer/src/kmc/src/kmlmp.ts

@@ -10,7 +10,7 @@ import { Command } from 'commander';
 import { PackageValidation, KmpCompiler } from '@keymanapp/kmc-package';
 import { SysExits } from './util/sysexits.js';
 import KEYMAN_VERSION from "@keymanapp/keyman-version";
-import { NodeCompilerCallbacks } from './messages/NodeCompilerCallbacks.js';
+import { NodeCompilerCallbacks } from './util/NodeCompilerCallbacks.js';
 
 let inputFilename: string;
 const program = new Command();

developer/src/kmc/src/util/KeymanSentry.ts

@@ -1,5 +1,5 @@
 import { KmnCompiler } from "@keymanapp/kmc-kmn";
-import { NodeCompilerCallbacks } from "../messages/NodeCompilerCallbacks.js";
+import { NodeCompilerCallbacks } from "./NodeCompilerCallbacks.js";
 import Sentry from "@sentry/node";
 import KEYMAN_VERSION from "@keymanapp/keyman-version";
 import { spawnChild } from "./spawnAwait.js";

developer/src/kmc/src/util/NodeCompilerCallbacks.ts

@@ -6,7 +6,7 @@ import { CompilerCallbacks, CompilerSchema, CompilerEvent,
          CompilerError,
          CompilerCallbackOptions,
          CompilerFileCallbacks} from '@keymanapp/common-types';
-import { InfrastructureMessages } from './messages.js';
+import { InfrastructureMessages } from '../messages/messages.js';
 import chalk from 'chalk';
 import supportsColor from 'supports-color';
 
@@ -174,7 +174,7 @@ export class NodeCompilerCallbacks implements CompilerCallbacks {
   }
 
   loadSchema(schema: CompilerSchema): Uint8Array {
-    let schemaPath = new URL('../util/' + schema + '.schema.json', import.meta.url);
+    let schemaPath = new URL('./' + schema + '.schema.json', import.meta.url);
     return fs.readFileSync(schemaPath);
   }
 

developer/src/kmc/test/test-messages.ts

@@ -3,7 +3,7 @@ import { assert } from 'chai';
 import { InfrastructureMessages } from '../src/messages/messages.js';
 import { verifyCompilerMessagesObject } from '@keymanapp/developer-test-helpers';
 import { makePathToFixture } from './helpers/index.js';
-import { NodeCompilerCallbacks } from '../src/messages/NodeCompilerCallbacks.js';
+import { NodeCompilerCallbacks } from './util/NodeCompilerCallbacks.js';
 import { CompilerErrorNamespace } from '@keymanapp/common-types';
 
 describe('InfrastructureMessages', function () {

web/common.inc.sh

@@ -22,7 +22,7 @@ function compile() {
   local COMPILE_TARGET="$1"
   local BUNDLE_FLAG="${2:-}"
 
-  tsc -b "${KEYMAN_ROOT}/web/src/$COMPILE_TARGET" -v
+  tsc -b "${KEYMAN_ROOT}/web/src/$COMPILE_TARGET"
 
   if [ -f "./build-bundler.js" ]; then
     node "./build-bundler.js" "$BUNDLE_FLAG"