spiegel-keyman/web/build.sh

#!/usr/bin/env bash
#
# Compiles the Keyman Engine for Web and its various end-products
#

set -eu

## START STANDARD BUILD SCRIPT INCLUDE
# adjust relative paths as necessary
THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")"
. "${THIS_SCRIPT%/*}/../resources/build/build-utils.sh"
## END STANDARD BUILD SCRIPT INCLUDE

. "$KEYMAN_ROOT/resources/shellHelperFunctions.sh"

# This script runs from its own folder
cd "$THIS_SCRIPT_PATH"

# ################################ Main script ################################

# Definition of global compile constants

# Ensures that we rely first upon the local npm-based install of Typescript.
# (Facilitates automated setup for build agents.)
# TODO: this should be removeable given set_keyman_standard_build_path does this in build-utils.sh (and relative paths are dodgy in $PATH!)
PATH="../node_modules/.bin:$PATH"

PREDICTIVE_TEXT_SOURCE="../common/predictive-text/unit_tests/in_browser/resources/models/simple-trie.js"
PREDICTIVE_TEXT_OUTPUT="src/test/manual/web/prediction-ui/simple-en-trie.js"

builder_describe "Builds Keyman Engine for Web (KMW)." \
  "@/common/web/keyman-version build" \
  "@/common/web/input-processor build" \
  "@/web/src/tools/building/sourcemap-root build" \
  "clean" \
  "configure" \
  "build" \
  "test             Runs unit tests.  Only $(builder_term test:web) is currently defined"  \
  ":embed           Builds the configuration of KMW used within the Keyman mobile apps" \
  ":engine          Builds all common code used by other targets" \
  ":web             Builds the website-oriented configuration of Keyman Engine for Web" \
  ":ui              Builds the desktop UI modules used by the $(builder_term :web) build product" \
  ":samples         Builds only sample & test pages found under src/samples and src/test" \
  ":tools           Builds related development + unit-test resources" \
  "--no-minify      Skips any minification steps in the build" \
  "--all            Sets action to run on KMW's submodules as well if appropriate (actions - $(builder_term test))"

# Possible TODO?
# "upload-symbols   Uploads build product to Sentry for error report symbolification.  Only defined for $(builder_term build:embed) and $(builder_term build:web)" \

builder_describe_outputs \
  configure         /node_modules \
  configure:embed   /node_modules \
  configure:engine  /node_modules \
  configure:web     /node_modules \
  configure:ui      /node_modules \
  configure:samples /node_modules \
  configure:tools   /node_modules \
  build:embed       /web/build/app/embed/release/keyman.js \
  build:engine      /web/build/engine/main/obj/keymanweb.js \
  build:web         /web/build/app/web/release/keymanweb.js \
  build:ui          /web/build/app/ui/release/kmwuibutton.js \
  build:samples     /web/$PREDICTIVE_TEXT_OUTPUT \
  build:tools       /web/build/tools/building/sourcemap-root/index.js

builder_parse "$@"

#### Build utility methods + definitions ####

# Build products for each main target.
EMBED_TARGETS=( "keyman.js" )
WEB_TARGETS=( "keymanweb.js" )
UI_TARGETS=( "kmwuibutton.js" "kmwuifloat.js" "kmwuitoggle.js" "kmwuitoolbar.js" )

: ${CLOSURECOMPILERPATH:=../node_modules/google-closure-compiler-java}
: ${JAVA:=java}

minifier="$CLOSURECOMPILERPATH/compiler.jar"

# We'd love to add the argument --source_map_include_content for distribution in the future,
# but Closure doesn't include the TS sources properly at this time.
#
# `checkTypes` is blocked b/c TypeScript can perform our type checking... and it causes an error
# with TypeScript's `extend` implementation (it doesn't recognize a constructor without manual edits).
# We also get a global `this` warning from the same.
#
# `checkVars` is blocked b/c Closure will otherwise fail on TypeScript namespacing, as each original TS
# source file will redeclare the namespace variable, despite being merged into a single file post-compilation.
#
# `jsDocMissingType` prevents errors on type documentation Closure thinks is missing.  TypeScript may not
# have the same requirements, and we trust TypeScript over Closure.
minifier_warnings="--jscomp_error=* --jscomp_off=lintChecks --jscomp_off=unusedLocalVariables --jscomp_off=globalThis --jscomp_off=checkTypes --jscomp_off=checkVars --jscomp_off=jsdocMissingType --jscomp_off=uselessCode --jscomp_off=missingRequire --jscomp_off=strictMissingRequire"

# We use these to prevent Closure from auto-inserting its own polyfills.  Turns out, they can break in the
# WebView used by Android API 19, which our app still supported when written.  Unsure if it can still occur
# in Android API 21, our current minimum.
#
# Also, we currently apply all needed polyfills either manually or during TS compilation; we don't need the extra,
# excess code.
minifier_lang_specs="--language_in ECMASCRIPT5 --language_out ECMASCRIPT5"
minifycmd="$JAVA -jar $minifier --compilation_level WHITESPACE_ONLY $minifier_warnings --generate_exports $minifier_lang_specs"

readonly minifier
readonly minifycmd

minified_sourcemap_cleaner="build/tools/building/sourcemap-root/index.mjs"

# Fails the build if a specified file does not exist.
assert_exists ( ) {
  if [[ ! -f $1 ]]; then
    builder_die "Build failed:  expected file ${COLOR_GREY}$1${COLOR_RESET} is missing."
  fi
}

# $1 - base file name
# $2 - output path
# $3 - optimization level
# $4 - extra path info to add to minified sourcemap "sourceRoot" property.
# $5 - additional output wrapper
minify ( ) {
    if [ $# -ge 4 ]; then
        cleanerOptions="--clean --sourceRoot $4"
    else
        cleanerOptions="--clean"
    fi

    if [ $# -ge 5 ]; then
        wrapper=$5
    else
        wrapper="%output%"
    fi

    local INPUT="$1"
    local INPUT_FILE="$(basename $1)"
    local INPUT_DIR="$(dirname $1)"
    local INPUT_SOURCEMAP="$INPUT_DIR/$INPUT_FILE.map"
    local OUTPUT="$2"
    local OUTPUT_FILE="$(basename $2)"
    local OUTPUT_SOURCEMAP="$(dirname $2)/$OUTPUT_FILE.map"

    # --source_map_location_mapping - maps paths on INPUT source maps for consumption by Closure.
    # ../../.. => keymanapp, ../.. => keymanapp/keyman.  We have TS root sources on 'keyman'.
    $minifycmd --source_map_input "$INPUT|$INPUT_SOURCEMAP" \
        --create_source_map "$OUTPUT_SOURCEMAP" --source_map_include_content \
        --source_map_location_mapping "$INPUT_DIR|../../.." \
        --js "$INPUT" --compilation_level $3 \
        --js_output_file "$OUTPUT" --warning_level VERBOSE --output_wrapper "$wrapper
//# sourceMappingURL=$INPUT_FILE.map"

    # Now to clean the source map.
    assert_exists "$OUTPUT"
    assert_exists "$OUTPUT_SOURCEMAP"

    # "Clean" the minified output sourcemaps.
    node $minified_sourcemap_cleaner "$INPUT_SOURCEMAP" "$OUTPUT_SOURCEMAP" $cleanerOptions
}

# Copies specified engine resources to the specified target's build output directories.
#
# ### Parameters
#
# * 1: `product`    the product's source path under src/
# * 2: `outputs`    an array of resource types to copy over
#
# ### Example
#
# ```bash
#   copy_resources app/web osk ui
# ```
copy_resources ( ) {
  local COMPILE_TARGET=$1
  shift

  local RESOURCES_TO_COPY=("$@")

  # We leave out obj here, as it's not a 'release' of any sort and
  # thus doesn't need to publish sources or resources.
  local CONFIGS=(debug)

  if ! builder_has_option --no-minify; then
    CONFIGS+=(release)
  fi

  builder_echo

  for CONFIG in "${CONFIGS[@]}";
  do
    local CONFIG_OUT_PATH=build/$COMPILE_TARGET/$CONFIG

    builder_echo "Copying resources to $CONFIG_OUT_PATH/src"

    for RESOURCE in "${RESOURCES_TO_COPY[@]}";
    do
      mkdir -p "$CONFIG_OUT_PATH/$RESOURCE"
      mkdir -p "$CONFIG_OUT_PATH/src/resources/$RESOURCE"

      builder_echo "- src/resources/$RESOURCE/ => $CONFIG_OUT_PATH/$RESOURCE"
      cp -Rf "src/resources/$RESOURCE"  "$CONFIG_OUT_PATH/"  >/dev/null
    done

    builder_echo
  done
}

# Copies specified source folders corresponding to the specified target's build
# output directories.
#
# ### Parameters
#
# * 1: `product`    the product's source path under src/
# * 2: `outputs`    an array of src/ subfolders to copy over
#
# ### Example
#
# ```bash
#   copy_sources app/web engine resources/osk
# ```
copy_sources ( ) {
  local COMPILE_TARGET=$1
  shift

  local SOURCES_TO_COPY=("$@")

  # We leave out obj here, as it's not a 'release' of any sort and
  # thus doesn't need to publish sources or resources.
  CONFIGS=(debug)

  if ! builder_has_option --no-minify; then
    CONFIGS+=(release)
  fi

  for CONFIG in "${CONFIGS[@]}";
  do
    local CONFIG_OUT_PATH=build/$COMPILE_TARGET/$CONFIG
    builder_echo "Copying $COMPILE_TARGET sources to $CONFIG_OUT_PATH/src"

    rm -rf "$CONFIG_OUT_PATH/src"
    mkdir -p "$CONFIG_OUT_PATH/src"
    echo $VERSION_PATCH > "$CONFIG_OUT_PATH/src/version.txt"

    for SOURCE_FOLDER in "${SOURCES_TO_COPY[@]}";
    do
      builder_echo "- src/$SOURCE_FOLDER/ => $CONFIG_OUT_PATH/src/$SOURCE_FOLDER/"
      mkdir -p "$CONFIG_OUT_PATH/src/$SOURCE_FOLDER"
      cp -Rf  "src/$SOURCE_FOLDER/"*    "$CONFIG_OUT_PATH/src/$SOURCE_FOLDER/"
    done

    builder_echo
  done
}

# Compiles compiled scripts from the first folder specified to the second folder specified.
#
# ### Parameters
#
# * 1: `src`      the folder containing scripts to be copied
# * 2: `dst`      the destination folder for the copy operation
# * 3: `scripts`  an array of filenames for expected compiled scripts
#
# ### Example
#
# ```bash
#   compile_and_minify build/app/web/obj build/app/web/debug keymanweb.js
# ```
copy_outputs ( ) {
  local src="$1"
  local dst="$2"

  shift
  shift

  local BASE_SCRIPTS=("$@")

  mkdir -p "$dst"

  for SCRIPTJS in "${BASE_SCRIPTS[@]}";
  do
    cp -Rf "$src/$SCRIPTJS"             "$dst/"
    cp -Rf "$src/$SCRIPTJS.map"         "$dst/"
  done
}

# Compiles all build products corresponding to the specified target.
#
# ### Parameters
#
# * 1: `product`    the product's source path under src/
#
# ### Example
#
# ```bash
#   compile app/embed
# ```
compile ( ) {
  local COMPILE_TARGET=$1
  tsc -b src/$COMPILE_TARGET -v || builder_die "Build command tsc -b src/$COMPILE_TARGET -v failed with exit code $?"
  builder_echo "$COMPILE_TARGET TypeScript compiled under build/$COMPILE_TARGET/obj"
}

# Finalizes all build products corresponding to the specified target.
# This should be called after `compile` for all `app/` targets.
#
# ### Parameters
#
# * 1: `product`    the product's source path under src/
# * 2: `outputs`    an array of expected output script files for the build
#
# ### Example
#
# ```bash
#   compile app/embed
#   finalize app/embed keyman.js
# ```
finalize ( ) {
  if [ $# -lt 2 ]; then
    builder_die "Scripting error: insufficient argument count!"
  fi

  local COMPILE_TARGET=$1
  local COMPILED_INTERMEDIATE_PATH=build/$COMPILE_TARGET/obj
  local DEBUG_OUT_PATH=build/$COMPILE_TARGET/debug
  local RELEASE_OUT_PATH=build/$COMPILE_TARGET/release

  shift

  local OUTPUT_SCRIPTS=("$@")

  # START:  debug output

  mkdir -p "$DEBUG_OUT_PATH"
  copy_outputs "$COMPILED_INTERMEDIATE_PATH" "$DEBUG_OUT_PATH" "${OUTPUT_SCRIPTS[@]}"

  builder_echo "Compiled $COMPILE_TARGET debug version saved under $DEBUG_OUT_PATH: ${OUTPUT_SCRIPTS[*]}"

  # START:  release output
  if ! builder_has_option --no-minify; then
    for SCRIPT in "${OUTPUT_SCRIPTS[@]}";
    do
      minify "$COMPILED_INTERMEDIATE_PATH/$SCRIPT" "$RELEASE_OUT_PATH/$SCRIPT" SIMPLE_OPTIMIZATIONS
    done

    builder_echo "Compiled $COMPILE_TARGET release version saved under $RELEASE_OUT_PATH: ${OUTPUT_SCRIPTS[*]}"
  else
    # The prior 'release' is now outdated:  delete it.
    rm -rf "$RELEASE_OUT_PATH"
  fi
}

#### Build action definitions ####

if builder_start_action configure; then
  verify_npm_setup

  if ! builder_has_option --no-minify; then
    # NPM install is required for the file to be present.
    if ! [ -f $minifier ];
    then
      builder_die "File $minifier does not exist:  have you set the environment variable \$CLOSURECOMPILERPATH?"
    fi
  fi

  builder_finish_action success configure
fi

## Clean actions

# Possible issue:  there's no clear rule to `clean` the engine, which is auto-built
# by build:embed and build:web.
#
# Some sort of command to run ONLY for a general `clean` (no target specified) would
# be perfect for that, I think.

if builder_start_action clean:engine; then
  src/engine/build.sh clean
  builder_finish_action success clean:engine
fi

if builder_start_action clean:embed; then
  rm -rf build/app/embed
  builder_finish_action success clean:embed
fi

if builder_start_action clean:web; then
  rm -rf build/app/web
  builder_finish_action success clean:web
fi

if builder_start_action clean:ui; then
  rm -rf build/app/ui
  builder_finish_action success clean:ui
fi

if builder_start_action clean:samples; then
  rm -f $PREDICTIVE_TEXT_OUTPUT

  builder_finish_action success clean:samples
fi

if builder_start_action clean:tools; then
  src/tools/build.sh clean

  builder_finish_action success clean:tools
fi

## Build actions

# Adds a simple version 'header' when there's a main engine build product.
if builder_has_action build:embed || \
   builder_has_action build:web || \
   builder_has_action build:ui; then

  builder_echo ""
  builder_echo purple "Compiling version ${VERSION}"
fi

builder_echo


if builder_start_action build:engine; then
  src/engine/build.sh build

  builder_finish_action success build:engine
fi

if builder_start_action build:embed; then
  compile app/embed
  finalize app/embed ${EMBED_TARGETS[@]}

  # The embedded version doesn't use UI modules.
  copy_resources app/embed osk
  copy_sources app/embed app/embed engine resources/osk

  builder_finish_action success build:embed

  # TODO:  handle this block somehow.

  # if [ $UPLOAD_EMBED_SENTRY = true ]; then  # upload-symbols:embed
  #   if [ $BUILD_DEBUG_EMBED = true ]; then
  #     ARTIFACT_FOLDER="release/unminified/embedded"
  #     pushd $EMBED_OUTPUT_NO_MINI
  #   else
  #     ARTIFACT_FOLDER="release/embedded"
  #     pushd $EMBED_OUTPUTs
  #   fi
  #   echo "Uploading to Sentry..."
  #   npm run sentry-cli -- releases files "$VERSION_GIT_TAG" upload-sourcemaps --strip-common-prefix $ARTIFACT_FOLDER --rewrite --ext js --ext map --ext ts || builder_die "Sentry upload failed."
  #   echo "Upload successful."
  #   popd
  # fi
fi

### -embed section complete.

if builder_start_action build:web; then
  compile app/web
  finalize app/web ${WEB_TARGETS[@]}

  # The testing pages need both osk & ui resources in the same place.
  copy_resources app/web osk ui
  copy_sources app/web app/web engine resources/osk

  builder_finish_action success build:web
fi

if builder_start_action build:ui; then
  compile app/ui
  finalize app/ui ${UI_TARGETS[@]}

  copy_resources app/ui ui
  copy_sources app/ui app/ui resources/ui

  builder_finish_action success build:ui
fi

if builder_start_action build:tools; then
  src/tools/build.sh
  builder_finish_action success build:tools
fi

if builder_start_action build:samples; then
  # Some test pages actually have build scripts.
  ./src/test/manual/embed/android-harness/build.sh  # is not yet builder-based.

  builder_echo "Copying samples & test page resources..."
  # Should probably be changed into a build script for the `prediction-ui` test page.
  cp "${PREDICTIVE_TEXT_SOURCE}" "${PREDICTIVE_TEXT_OUTPUT}"

  # Which could then have a parallel script for `prediction-mtnt` that downloads + extracts
  # the current MTNT model.

  builder_finish_action success build:samples;
fi

if builder_start_action test:web; then
  if builder_has_option --all; then
    ./test.sh
  else
    ./test.sh :engine
  fi

  builder_finish_action success test:web
fi

# TODO:  handle the block below somehow.

# # We can only upload 'web' / 'native' artifacts after ALL are done compiling.
# if [ $UPLOAD_WEB_SENTRY = true ]; then
#     pushd $WEB_OUTPUT
#     echo "Uploading to Sentry..."
#     npm run sentry-cli -- releases files "$VERSION_GIT_TAG" upload-sourcemaps --strip-common-prefix release/web/ --rewrite --ext js --ext map --ext ts || builder_die "Sentry upload failed."
#     echo "Upload successful."
#     popd
# fi