From fa3980f598e45d8a72b571c11a1fdeca266dae43 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 12 Jul 2023 10:20:35 +0700 Subject: [PATCH] feat(developer): deploy kmc as ES module Deploy kmc as an ES module, forced now by top-level await that we use in the sentry load. Moved NodeCompilerCallbacks to same util folder as schemas are found in, to avoid rewriting schema loading with imports in this PR. It belongs better there than messages/ anyway. Distributed files are now .mjs. Removed --enable-source-maps from launchers as logging will be managed with sentry anyway. --- developer/src/inst/node/kmc.cmd | 2 +- developer/src/inst/node/kmlmc.cmd | 2 +- developer/src/inst/node/kmlmi.cmd | 2 +- developer/src/inst/node/kmlmp.cmd | 2 +- developer/src/kmc-model/build.sh | 1 - developer/src/kmc/build-bundler.js | 32 +++++++++++++++++++ developer/src/kmc/build.sh | 32 +++++++++++++------ developer/src/kmc/src/commands/analyze.ts | 2 +- developer/src/kmc/src/commands/build.ts | 2 +- .../kmc/src/commands/buildTestData/index.ts | 2 +- developer/src/kmc/src/kmlmc.ts | 2 +- developer/src/kmc/src/kmlmi.ts | 2 +- developer/src/kmc/src/kmlmp.ts | 2 +- developer/src/kmc/src/util/KeymanSentry.ts | 2 +- .../NodeCompilerCallbacks.ts | 4 +-- developer/src/kmc/test/test-messages.ts | 2 +- web/common.inc.sh | 2 +- 17 files changed, 69 insertions(+), 26 deletions(-) create mode 100644 developer/src/kmc/build-bundler.js rename developer/src/kmc/src/{messages => util}/NodeCompilerCallbacks.ts (97%) diff --git a/developer/src/inst/node/kmc.cmd b/developer/src/inst/node/kmc.cmd index c1ac96270f..c529011171 100644 --- a/developer/src/inst/node/kmc.cmd +++ b/developer/src/inst/node/kmc.cmd @@ -1,4 +1,4 @@ @rem This script avoids path dependencies for node for distribution @rem with Keyman Developer. When used on platforms other than Windows, @rem node can be used directly with the compiler (`npm link` will setup). -@"%~dp0\node.js\node.exe" --enable-source-maps "%~dp0\kmc\kmc.cjs" %* +@"%~dp0\node.js\node.exe" "%~dp0\kmc\kmc.mjs" %* diff --git a/developer/src/inst/node/kmlmc.cmd b/developer/src/inst/node/kmlmc.cmd index 81b87c0f9e..be3bdf06e0 100644 --- a/developer/src/inst/node/kmlmc.cmd +++ b/developer/src/inst/node/kmlmc.cmd @@ -1,4 +1,4 @@ @rem This script avoids path dependencies for node for distribution @rem with Keyman Developer. When used on platforms other than Windows, @rem node can be used directly with the compiler (`npm link` will setup). -@"%~dp0\node.js\node.exe" --enable-source-maps "%~dp0\kmc\kmlmc.cjs" %* +@"%~dp0\node.js\node.exe" "%~dp0\kmc\kmlmc.mjs" %* diff --git a/developer/src/inst/node/kmlmi.cmd b/developer/src/inst/node/kmlmi.cmd index c02a72891c..e674fe4d2c 100644 --- a/developer/src/inst/node/kmlmi.cmd +++ b/developer/src/inst/node/kmlmi.cmd @@ -1,4 +1,4 @@ @rem This script avoids path dependencies for node for distribution @rem with Keyman Developer. When used on platforms other than Windows, @rem node can be used directly with the compiler (`npm link` will setup). -@"%~dp0\node.js\node.exe" --enable-source-maps "%~dp0\kmc\kmlmi.cjs" %* +@"%~dp0\node.js\node.exe" "%~dp0\kmc\kmlmi.mjs" %* diff --git a/developer/src/inst/node/kmlmp.cmd b/developer/src/inst/node/kmlmp.cmd index 8994cf4db3..2062ae091c 100644 --- a/developer/src/inst/node/kmlmp.cmd +++ b/developer/src/inst/node/kmlmp.cmd @@ -1,4 +1,4 @@ @rem This script avoids path dependencies for node for distribution @rem with Keyman Developer. When used on platforms other than Windows, @rem node can be used directly with the compiler (`npm link` will setup). -@"%~dp0\node.js\node.exe" --enable-source-maps "%~dp0\kmc\kmlmp.cjs" %* +@"%~dp0\node.js\node.exe" "%~dp0\kmc\kmlmp.mjs" %* diff --git a/developer/src/kmc-model/build.sh b/developer/src/kmc-model/build.sh index ecf58e20e4..2e73366b6b 100755 --- a/developer/src/kmc-model/build.sh +++ b/developer/src/kmc-model/build.sh @@ -32,7 +32,6 @@ builder_parse "$@" #------------------------------------------------------------------------------------------------------------------- function do_build() { - mkdir -p build/cjs-src npm run build } diff --git a/developer/src/kmc/build-bundler.js b/developer/src/kmc/build-bundler.js new file mode 100644 index 0000000000..5d55ee4be3 --- /dev/null +++ b/developer/src/kmc/build-bundler.js @@ -0,0 +1,32 @@ +/* + * Bundle kmc as esm with appropriate node modules and banner + */ + +import esbuild from 'esbuild'; + +await esbuild.build({ + entryPoints: [ + 'build/src/kmc.js', + 'build/src/kmlmc.js', + 'build/src/kmlmi.js', + 'build/src/kmlmp.js', + ], + bundle: true, + format: 'esm', + platform: 'node', + target: 'es2022', + outdir: 'build/dist/', + + // We want a .mjs extension to force node into ESM module mode + outExtension: { '.js': '.mjs' }, + + // Thunk for external modules such as Commander that are still CJS and still + // use require, __filename, __dirname + banner: { + js: ` + const require = (await import("node:module")).createRequire(import.meta.url); + const __filename = (await import("node:url")).fileURLToPath(import.meta.url); + const __dirname = (await import("node:path")).dirname(__filename); + `, + }, +}); diff --git a/developer/src/kmc/build.sh b/developer/src/kmc/build.sh index 2ec2c57377..d5d5e80a66 100755 --- a/developer/src/kmc/build.sh +++ b/developer/src/kmc/build.sh @@ -56,13 +56,20 @@ fi function copy_schemas() { # We need the schema file at runtime and bundled, so always copy it for all actions except `clean` + local schemas=( + "$KEYMAN_ROOT/resources/standards-data/ldml-keyboards/techpreview/ldml-keyboard.schema.json" + "$KEYMAN_ROOT/resources/standards-data/ldml-keyboards/techpreview/ldml-keyboardtest.schema.json" + "$KEYMAN_ROOT/common/schemas/kvks/kvks.schema.json" + "$KEYMAN_ROOT/common/schemas/kpj/kpj.schema.json" + "$KEYMAN_ROOT/common/schemas/kpj-9.0/kpj-9.0.schema.json" + "$KEYMAN_ROOT/common/schemas/displaymap/displaymap.schema.json" + ) + mkdir -p "$THIS_SCRIPT_PATH/build/src/util/" - cp "$KEYMAN_ROOT/resources/standards-data/ldml-keyboards/techpreview/ldml-keyboard.schema.json" "$THIS_SCRIPT_PATH/build/src/util/" - cp "$KEYMAN_ROOT/resources/standards-data/ldml-keyboards/techpreview/ldml-keyboardtest.schema.json" "$THIS_SCRIPT_PATH/build/src/util/" - cp "$KEYMAN_ROOT/common/schemas/kvks/kvks.schema.json" "$THIS_SCRIPT_PATH/build/src/util/" - cp "$KEYMAN_ROOT/common/schemas/kpj/kpj.schema.json" "$THIS_SCRIPT_PATH/build/src/util/" - cp "$KEYMAN_ROOT/common/schemas/kpj-9.0/kpj-9.0.schema.json" "$THIS_SCRIPT_PATH/build/src/util/" - cp "$KEYMAN_ROOT/common/schemas/displaymap/displaymap.schema.json" "$THIS_SCRIPT_PATH/build/src/util/" + cp "${schemas[@]}" "$THIS_SCRIPT_PATH/build/src/util/" + + mkdir -p "$THIS_SCRIPT_PATH/build/dist/" + cp "${schemas[@]}" "$THIS_SCRIPT_PATH/build/dist/" } #------------------------------------------------------------------------------------------------------------------- @@ -94,16 +101,21 @@ fi #------------------------------------------------------------------------------------------------------------------- if builder_start_action bundle; then - copy_schemas if ! builder_has_option --build-path; then builder_finish_action "Parameter --build-path is required" bundle exit 64 fi - mkdir -p build/cjs-src - npm run bundle - cp build/cjs-src/* "$BUILD_PATH" + rm -rf build/dist + mkdir -p build/dist + node build-bundler.js + + # Manually copy over kmcmplib module and schemas + copy_schemas + cp ../kmc-kmn/build/src/import/kmcmplib/wasm-host.wasm build/dist/ + + cp build/dist/* "$BUILD_PATH" builder_finish_action success bundle fi diff --git a/developer/src/kmc/src/commands/analyze.ts b/developer/src/kmc/src/commands/analyze.ts index 3a71d2008e..d71f58afe8 100644 --- a/developer/src/kmc/src/commands/analyze.ts +++ b/developer/src/kmc/src/commands/analyze.ts @@ -1,7 +1,7 @@ import * as fs from 'fs'; import * as path from 'path'; import { Command, Option } from 'commander'; -import { NodeCompilerCallbacks } from '../messages/NodeCompilerCallbacks.js'; +import { NodeCompilerCallbacks } from '../util/NodeCompilerCallbacks.js'; import { InfrastructureMessages } from '../messages/messages.js'; import { CompilerCallbacks, CompilerLogLevel } from '@keymanapp/common-types'; import { AnalyzeOskCharacterUse, AnalyzeOskRewritePua } from '@keymanapp/kmc-analyze'; diff --git a/developer/src/kmc/src/commands/build.ts b/developer/src/kmc/src/commands/build.ts index ddd916f34d..fade2070dd 100644 --- a/developer/src/kmc/src/commands/build.ts +++ b/developer/src/kmc/src/commands/build.ts @@ -2,7 +2,7 @@ import * as fs from 'fs'; import { Command } from 'commander'; import { buildActivities } from './buildClasses/buildActivities.js'; import { BuildProject } from './buildClasses/BuildProject.js'; -import { NodeCompilerCallbacks } from '../messages/NodeCompilerCallbacks.js'; +import { NodeCompilerCallbacks } from '../util/NodeCompilerCallbacks.js'; import { InfrastructureMessages } from '../messages/messages.js'; import { CompilerFileCallbacks, CompilerOptions, KeymanFileTypes } from '@keymanapp/common-types'; import { BaseOptions } from '../util/baseOptions.js'; diff --git a/developer/src/kmc/src/commands/buildTestData/index.ts b/developer/src/kmc/src/commands/buildTestData/index.ts index d3c2e98d94..b50c63c1d6 100644 --- a/developer/src/kmc/src/commands/buildTestData/index.ts +++ b/developer/src/kmc/src/commands/buildTestData/index.ts @@ -2,7 +2,7 @@ import * as fs from 'fs'; import * as path from 'path'; import * as kmcLdml from '@keymanapp/kmc-ldml'; import { CompilerBaseOptions, CompilerCallbacks, defaultCompilerOptions, LDMLKeyboardTestDataXMLSourceFile, LDMLKeyboardXMLSourceFileReader } from '@keymanapp/common-types'; -import { NodeCompilerCallbacks } from '../../messages/NodeCompilerCallbacks.js'; +import { NodeCompilerCallbacks } from '../../util/NodeCompilerCallbacks.js'; import { fileURLToPath } from 'url'; export function buildTestData(infile: string, options: CompilerBaseOptions) { diff --git a/developer/src/kmc/src/kmlmc.ts b/developer/src/kmc/src/kmlmc.ts index c021fb9384..1ce171fd15 100644 --- a/developer/src/kmc/src/kmlmc.ts +++ b/developer/src/kmc/src/kmlmc.ts @@ -8,7 +8,7 @@ import { Command } from 'commander'; import { compileModel } from '@keymanapp/kmc-model'; import { SysExits } from './util/sysexits.js'; import KEYMAN_VERSION from "@keymanapp/keyman-version"; -import { NodeCompilerCallbacks } from './messages/NodeCompilerCallbacks.js'; +import { NodeCompilerCallbacks } from './util/NodeCompilerCallbacks.js'; let inputFilename: string; const program = new Command(); diff --git a/developer/src/kmc/src/kmlmi.ts b/developer/src/kmc/src/kmlmi.ts index f64ebb66f6..159dad38e0 100644 --- a/developer/src/kmc/src/kmlmi.ts +++ b/developer/src/kmc/src/kmlmi.ts @@ -9,7 +9,7 @@ import { KmpCompiler, PackageValidation } from '@keymanapp/kmc-package'; import { ModelInfoOptions, writeMergedModelMetadataFile } from '@keymanapp/kmc-model-info'; import { SysExits } from './util/sysexits.js'; import KEYMAN_VERSION from "@keymanapp/keyman-version"; -import { NodeCompilerCallbacks } from './messages/NodeCompilerCallbacks.js'; +import { NodeCompilerCallbacks } from './util/NodeCompilerCallbacks.js'; let inputFilename: string; const program = new Command(); diff --git a/developer/src/kmc/src/kmlmp.ts b/developer/src/kmc/src/kmlmp.ts index a866b231f5..d02420eeca 100644 --- a/developer/src/kmc/src/kmlmp.ts +++ b/developer/src/kmc/src/kmlmp.ts @@ -10,7 +10,7 @@ import { Command } from 'commander'; import { PackageValidation, KmpCompiler } from '@keymanapp/kmc-package'; import { SysExits } from './util/sysexits.js'; import KEYMAN_VERSION from "@keymanapp/keyman-version"; -import { NodeCompilerCallbacks } from './messages/NodeCompilerCallbacks.js'; +import { NodeCompilerCallbacks } from './util/NodeCompilerCallbacks.js'; let inputFilename: string; const program = new Command(); diff --git a/developer/src/kmc/src/util/KeymanSentry.ts b/developer/src/kmc/src/util/KeymanSentry.ts index 2d0844f04d..a2a466b83d 100644 --- a/developer/src/kmc/src/util/KeymanSentry.ts +++ b/developer/src/kmc/src/util/KeymanSentry.ts @@ -1,5 +1,5 @@ import { KmnCompiler } from "@keymanapp/kmc-kmn"; -import { NodeCompilerCallbacks } from "../messages/NodeCompilerCallbacks.js"; +import { NodeCompilerCallbacks } from "./NodeCompilerCallbacks.js"; import Sentry from "@sentry/node"; import KEYMAN_VERSION from "@keymanapp/keyman-version"; import { spawnChild } from "./spawnAwait.js"; diff --git a/developer/src/kmc/src/messages/NodeCompilerCallbacks.ts b/developer/src/kmc/src/util/NodeCompilerCallbacks.ts similarity index 97% rename from developer/src/kmc/src/messages/NodeCompilerCallbacks.ts rename to developer/src/kmc/src/util/NodeCompilerCallbacks.ts index 376ef5b6d9..53cd80243a 100644 --- a/developer/src/kmc/src/messages/NodeCompilerCallbacks.ts +++ b/developer/src/kmc/src/util/NodeCompilerCallbacks.ts @@ -6,7 +6,7 @@ import { CompilerCallbacks, CompilerSchema, CompilerEvent, CompilerError, CompilerCallbackOptions, CompilerFileCallbacks} from '@keymanapp/common-types'; -import { InfrastructureMessages } from './messages.js'; +import { InfrastructureMessages } from '../messages/messages.js'; import chalk from 'chalk'; import supportsColor from 'supports-color'; @@ -174,7 +174,7 @@ export class NodeCompilerCallbacks implements CompilerCallbacks { } loadSchema(schema: CompilerSchema): Uint8Array { - let schemaPath = new URL('../util/' + schema + '.schema.json', import.meta.url); + let schemaPath = new URL('./' + schema + '.schema.json', import.meta.url); return fs.readFileSync(schemaPath); } diff --git a/developer/src/kmc/test/test-messages.ts b/developer/src/kmc/test/test-messages.ts index d621cefbe9..57a460411d 100644 --- a/developer/src/kmc/test/test-messages.ts +++ b/developer/src/kmc/test/test-messages.ts @@ -3,7 +3,7 @@ import { assert } from 'chai'; import { InfrastructureMessages } from '../src/messages/messages.js'; import { verifyCompilerMessagesObject } from '@keymanapp/developer-test-helpers'; import { makePathToFixture } from './helpers/index.js'; -import { NodeCompilerCallbacks } from '../src/messages/NodeCompilerCallbacks.js'; +import { NodeCompilerCallbacks } from './util/NodeCompilerCallbacks.js'; import { CompilerErrorNamespace } from '@keymanapp/common-types'; describe('InfrastructureMessages', function () { diff --git a/web/common.inc.sh b/web/common.inc.sh index 1dc057b1e9..e1594c911c 100644 --- a/web/common.inc.sh +++ b/web/common.inc.sh @@ -22,7 +22,7 @@ function compile() { local COMPILE_TARGET="$1" local BUNDLE_FLAG="${2:-}" - tsc -b "${KEYMAN_ROOT}/web/src/$COMPILE_TARGET" -v + tsc -b "${KEYMAN_ROOT}/web/src/$COMPILE_TARGET" if [ -f "./build-bundler.js" ]; then node "./build-bundler.js" "$BUNDLE_FLAG"