Merge pull request #10413 from keymanapp/chore/web/sourcemap-issues

chore(web): worker sourcemap cleanup, mobile app webview sourcemap compat

android/KMEA/app/src/main/java/com/keyman/engine/KMManager.java

@@ -293,9 +293,7 @@ public final class KMManager {
   protected static final String KMFilename_KeyboardHtml = "keyboard.html";
   protected static final String KMFilename_KeyboardHtml_Legacy = "keyboard.es5.html";
   protected static final String KMFilename_JSEngine = "keymanweb-webview.js";
-  protected static final String KMFilename_JSEngine_Sourcemap = "keymanweb-webview.js.map";
   protected static final String KMFilename_JSLegacyEngine = "keymanweb-webview.es5.js";
-  protected static final String KMFilename_JSLegacyEngine_Sourcemap = "keymanweb-webview.es5.js.map";
   protected static final String KMFilename_JSSentry = "sentry.min.js";
   protected static final String KMFilename_JSSentryInit = "keyman-sentry.js";
   protected static final String KMFilename_AndroidHost = "android-host.js";
@@ -867,17 +865,11 @@ public final class KMManager {
         copyAssetWithRename(context, KMFilename_KeyboardHtml_Legacy, KMFilename_KeyboardHtml, "", true);
 
         copyAsset(context, KMFilename_JSLegacyEngine, "", true);
-        if (KMManager.isDebugMode()) {
-          copyAsset(context, KMFilename_JSLegacyEngine_Sourcemap, "", true);
-        }
       } else {
         copyAsset(context, KMFilename_KeyboardHtml, "", true);
 
         // For versions of Chrome with full ES6 support, we use the ES6 artifact.
         copyAsset(context, KMFilename_JSEngine, "", true);
-        if (KMManager.isDebugMode()) {
-          copyAsset(context, KMFilename_JSEngine_Sourcemap, "", true);
-        }
       }
 
       // Is still built targeting ES5.

common/web/lm-worker/build.sh

@@ -28,10 +28,13 @@ LIB=./build/lib
 
 bundle_cmd="node ../es-bundling/build/common-bundle.mjs"
 
+SRCMAP_CLEANER="node $KEYMAN_ROOT/web/build/tools/building/sourcemap-root/index.js"
+
 ################################ Main script ################################
 
 builder_describe \
   "Compiles the Language Modeling Layer for common use in predictive text and autocorrective applications." \
+  "@/web/src/tools/building/sourcemap-root" \
   "@/common/web/keyman-version" \
   "@/common/web/es-bundling" \
   "@/common/models/wordbreakers" \
@@ -49,12 +52,24 @@ function do_build() {
   tsc -b $builder_verbose || builder_die "Could not build worker."
 
   $bundle_cmd build/obj/worker-main.js \
-    --out $INTERMEDIATE/worker-main.es5.js
+    --out $INTERMEDIATE/worker-main.es5.js \
+    --sourceRoot '@keymanapp/keyman/common/web/lm-worker/src/main'
+
+  $SRCMAP_CLEANER \
+    $INTERMEDIATE/worker-main.es5.js.map \
+    $INTERMEDIATE/worker-main.es5.js.map \
+    --clean
 
   $bundle_cmd build/obj/worker-main.js \
     --out $INTERMEDIATE/worker-main.min.es5.js \
     --minify \
-    --profile build/filesize-profile.es5.log
+    --profile build/filesize-profile.es5.log \
+    --sourceRoot '@keymanapp/keyman/common/web/lm-worker/src/main'
+
+  $SRCMAP_CLEANER \
+    $INTERMEDIATE/worker-main.min.es5.js.map \
+    $INTERMEDIATE/worker-main.min.es5.js.map \
+    --clean
 
   EXT_FLAGS=
   if builder_has_option --ci; then
@@ -79,13 +94,25 @@ function do_build() {
   # The ES6 target needs no polyfills - we go straight to the wrapped version.
   $bundle_cmd src/main/worker-main.ts \
     --out $INTERMEDIATE/worker-main.js \
-    --target "es6"
+    --target "es6" \
+    --sourceRoot '@keymanapp/keyman/common/web/lm-worker/src/main'
+
+  $SRCMAP_CLEANER \
+    $INTERMEDIATE/worker-main.js.map \
+    $INTERMEDIATE/worker-main.js.map \
+    --clean
 
   $bundle_cmd src/main/worker-main.ts \
     --out $INTERMEDIATE/worker-main.min.js \
     --minify \
     --profile build/filesize-profile.log \
-    --target "es6"
+    --target "es6" \
+    --sourceRoot '@keymanapp/keyman/common/web/lm-worker/src/main'
+
+  $SRCMAP_CLEANER \
+    $INTERMEDIATE/worker-main.min.js.map \
+    $INTERMEDIATE/worker-main.min.js.map \
+    --clean
 
   node build-wrapper.js $INTERMEDIATE/worker-main.js \
     --out $LIB/worker-main.wrapped.js \

ios/engine/KMEI/KeymanEngine/Classes/Resource Management/Storage.swift

@@ -236,13 +236,6 @@ extension Storage {
                      resourceName: "keyman-sentry.js",
                      dstDir: baseDir,
                      excludeFromBackup: true)
-    // For debug compilations - IF we have a sourcemap file, copy that over too.
-    if bundle.url(forResource: "keymanweb-webview.js.map", withExtension: nil) != nil {
-      try Storage.copy(from: bundle,
-                       resourceName: "keymanweb-webview.js.map",
-                       dstDir: baseDir,
-                       excludeFromBackup: true)
-    }
     try Storage.copy(from: bundle,
                      resourceName: "kmwosk.css",
                      dstDir: baseDir,

ios/engine/build.sh

@@ -111,12 +111,6 @@ function update_bundle ( ) {
   cp "$KMW_RESOURCES/osk/keymanweb-osk.ttf"     "$BUNDLE_PATH/keymanweb-osk.ttf"
   cp "$KMW_PRODUCT/keymanweb-webview.js"        "$BUNDLE_PATH/keymanweb-webview.js"
 
-  if builder_is_debug_build; then
-    cp "$KMW_PRODUCT/keymanweb-webview.js.map"  "$BUNDLE_PATH/keymanweb-webview.js.map"
-  else
-    rm -f "$BUNDLE_PATH/keymanweb-webview.js.map"
-  fi
-
   cp "$KEYMAN_ROOT/node_modules/@sentry/browser/build/bundle.min.js" "$BUNDLE_PATH/sentry.min.js"
   cp "$KEYMAN_ROOT/common/web/sentry-manager/build/lib/index.js"     "$BUNDLE_PATH/keyman-sentry.js"
 }

web/src/app/webview/build.sh

@@ -72,12 +72,18 @@ compile_and_copy() {
   cp -R "$KEYMAN_ROOT/web/src/resources/osk/." "$KEYMAN_ROOT/web/build/app/resources/osk/"
 
   # Clean the sourcemaps of .. and . components
-  for sourcemap in "$KEYMAN_ROOT/web/build/$SUBPROJECT_NAME/debug/"*.map; do
-    node "$KEYMAN_ROOT/web/build/tools/building/sourcemap-root/index.js" null "$sourcemap" --clean
+  for script in "$KEYMAN_ROOT/web/build/$SUBPROJECT_NAME/debug/"*.js; do
+    sourcemap="$script.map"
+    node "$KEYMAN_ROOT/web/build/tools/building/sourcemap-root/index.js" \
+      "$script" "$sourcemap" --clean --inline
   done
 
-  for sourcemap in "$KEYMAN_ROOT/web/build/$SUBPROJECT_NAME/release/"*.map; do
-    node "$KEYMAN_ROOT/web/build/tools/building/sourcemap-root/index.js" null "$sourcemap" --clean
+  # Do NOT inline sourcemaps for release builds - we don't want them to affect
+  # load time.
+  for script in "$KEYMAN_ROOT/web/build/$SUBPROJECT_NAME/release/"*.js; do
+    sourcemap="$script.map"
+    node "$KEYMAN_ROOT/web/build/tools/building/sourcemap-root/index.js" \
+      "$script" "$sourcemap" --clean
   done
 
   node map-polyfill-bundler.js

web/src/tools/building/sourcemap-root/index.ts

@@ -1,13 +1,19 @@
 import SourcemapRemapper from "@keymanapp/sourcemap-path-remapper"
+import convertSourceMap from 'convert-source-map'; // Transforms sourcemaps among various common formats.
+                                                   // Base64, stringified-JSON, end-of-file comment...
+import fs from 'fs';
 
 function displayHelp() {
   console.log("KeymanWeb's sourcemap-cleansing tool.  This tool is designed to produce clean filepaths");
   console.log("in the sourcemaps for ease of reference in browser and Sentry, copying the clean paths TS");
   console.log("is able to provide, whereas Closure's minification is not.");
   console.log("");
-  console.log("Usage:  node sourcemap-root <source.map> <dest.map> [-c|--clean] [-s|--sourceRoot <path>]");
+  console.log("Usage:  node sourcemap-root <source.js> <source.map> [-c|--clean] [-s|--sourceRoot <path>]");
   console.log("        -c|--clean Runs source-filepath cleaning operations.");
+  console.log("        -i|--inline Inlines <source.map> as a comment within <source.js>.");
   console.log("        -s|--sourceRoot <path> Sets the input sourcemap's 'sourceRoot' to <path>.");
+  console.log("");
+  console.log("        <source.js> is ignored unless the --inline option is specified.");
 }
 
 // By default, any node-based process has two command-line args:
@@ -38,15 +44,17 @@ function assert_is_map(filename: string) {
     process.exit(1);
   }
 }
-// Verify the first parameter is a sourcemap filepath.
-// let sourceFile = process.argv[2]; // Used to be necessary, but no longer is.
-let destFile = process.argv[3];
 
-assert_is_map(destFile);
+// Verify the second parameter is a sourcemap filepath.
+let scriptFile = process.argv[2];
+let mapFile = process.argv[3];
+
+assert_is_map(mapFile);
 
 // Now to process any additional flags.
 var sourceRoot = "";
 let shouldClean = false;
+let shouldInline = false;
 
 // Starting at [5] - optional command-line arguments
 for(let procArgIndex = 4; procArgIndex < process.argv.length; procArgIndex++) {
@@ -62,6 +70,10 @@ for(let procArgIndex = 4; procArgIndex < process.argv.length; procArgIndex++) {
     case "--clean":
       shouldClean = true;
       break;
+    case "-i":
+    case "--inline":
+      shouldInline = true;
+      break;
     case "-s":
     case "--sourceRoot":
       procArgIndex++;
@@ -72,7 +84,7 @@ for(let procArgIndex = 4; procArgIndex < process.argv.length; procArgIndex++) {
   }
 }
 
-let srcMap = SourcemapRemapper.fromFile(destFile);
+let srcMap = SourcemapRemapper.fromFile(mapFile);
 
 if(shouldClean) {
   // Keeps "@keymanapp/keyman" as sourceRoot, but prepends all `sources` entries with
@@ -84,4 +96,11 @@ if(sourceRoot) {
   srcMap.sourceRoot = sourceRoot;
 }
 
-srcMap.toFile(destFile);
+srcMap.toFile(mapFile);
+
+if(shouldInline) {
+  const mapToInline = SourcemapRemapper.fromFile(mapFile).sourceRoot;
+  const inlineableComment = convertSourceMap.fromObject(mapToInline).toComment();
+
+  fs.appendFileSync(scriptFile, `\n${inlineableComment}`);
+}