spiegel-keyman/web/build.sh

#!/usr/bin/env bash
#
# Compiles the Keyman Engine for Web and its various end-products
#

# set -x
set -eu

## START STANDARD BUILD SCRIPT INCLUDE
# adjust relative paths as necessary
THIS_SCRIPT="$(greadlink -f "${BASH_SOURCE[0]}" 2>/dev/null || readlink -f "${BASH_SOURCE[0]}")"
. "$(dirname "$THIS_SCRIPT")/../resources/build/build-utils.sh"
## END STANDARD BUILD SCRIPT INCLUDE

. "$KEYMAN_ROOT/resources/shellHelperFunctions.sh"

# This script runs from its own folder
cd "$THIS_SCRIPT_PATH"

# ################################ Main script ################################

# Definition of global compile constants

UI="app/ui"
WEB="app/web"
EMBEDDED="app/embed"

BUILD_BASE="build"

DEBUG="debug"
RELEASE="release"
INTERMEDIATE="obj"

# Composites and outputs the output path corresponding to the build configuration
# specified by the parameters.
#
# ### Parameters
#
# * 1: - build product (app/embed, app/web, app/ui, engine)
# * 2: (optional) - build stage / config (obj, debug, release)
#
# ### Example
#
# ```bash
#   cp index.js "$(output_path app/web debug)/index.js"
# ```
#
# The block above would copy index.js into the build output folder for app/web's debug
# product.
#
# ``` bash
#   rm -rf "$(output_path app/web)"
# ```
#
# The block above is useful for deleting all app/web build products as part of a `clean`
# action.
#
# ### Other Notes
#
# In the future, we may opt to move $INTERMEDIATE stuff underneath both $DEBUG and $RELEASE,
# making it a third param.  This is currently unclear, but if so, we'd do
# $DEBUG/$INTERMEDIATE and $RELEASE/$INTERMEDIATE via a third argument.
output_path ( ) {
  if [ $# -lt 1 ]; then
    builder_die "Insufficient argument count!"
  elif [ $# -eq 1 ]; then
    # Used by clean:<target> actions
    echo "$BUILD_BASE/$1"
  else
    echo "$BUILD_BASE/$1/$2"
  fi
}

SOURCE="src"

SENTRY_RELEASE_VERSION="release@$VERSION_WITH_TAG"

# Ensures that we rely first upon the local npm-based install of Typescript.
# (Facilitates automated setup for build agents.)
PATH="../node_modules/.bin:$PATH"

compiler="npm run tsc --"
compilecmd="$compiler"

PREDICTIVE_TEXT_SOURCE="../common/predictive-text/unit_tests/in_browser/resources/models/simple-trie.js"
PREDICTIVE_TEXT_OUTPUT="src/test/manual/web/prediction-ui/simple-en-trie.js"

builder_check_color "$@"

builder_describe "Builds Keyman Engine for Web (KMW)." \
  "@../common/web/keyman-version build" \
  "@../common/web/input-processor build" \
  "@src/tools/building/sourcemap-root build" \
  "clean" \
  "configure" \
  "build" \
  "test             Runs unit tests.  Only $(builder_term test:web) is currently defined"  \
  ":embed           Builds the configuration of KMW used within the Keyman mobile apps" \
  ":engine          Builds all common code used by other targets" \
  ":web             Builds the website-oriented configuration of Keyman Engine for Web" \
  ":ui              Builds the desktop UI modules used by the $(builder_term :web) build product" \
  ":samples         Builds only sample & test pages found under src/samples and src/test" \
  ":tools           Builds related development + unit-test resources" \
  "--no-minify      Skips any minification steps in the build" \
  "--all            Sets action to run on KMW's submodules as well if appropriate (actions - $(builder_term test))"

# Possible TODO?
# "upload-symbols   Uploads build product to Sentry for error report symbolification.  Only defined for $(builder_term build:embed) and $(builder_term build:web)" \

builder_describe_outputs \
  configure         ../node_modules \
  configure:embed   ../node_modules \
  configure:engine  ../node_modules \
  configure:web     ../node_modules \
  configure:ui      ../node_modules \
  configure:samples ../node_modules \
  configure:tools   ../node_modules \
  build:embed       $(output_path $EMBEDDED $RELEASE)/keyman.js \
  build:web         $(output_path $WEB $RELEASE)/keymanweb.js \
  build:ui          $(output_path $UI $RELEASE)/kmwuibutton.js \
  build:samples     $PREDICTIVE_TEXT_OUTPUT
# Deliberately excluding build:tools b/c its script provides the definitions.

builder_parse "$@"

#### Build utility methods + definitions ####

# Build products for each main target.
EMBED_TARGETS=( "keyman.js" )
WEB_TARGETS=( "keymanweb.js" )
UI_TARGETS=( "kmwuibutton.js" "kmwuifloat.js" "kmwuitoggle.js" "kmwuitoolbar.js" )

: ${CLOSURECOMPILERPATH:=../node_modules/google-closure-compiler-java}
: ${JAVA:=java}

minifier="$CLOSURECOMPILERPATH/compiler.jar"

# We'd love to add the argument --source_map_include_content for distribution in the future,
# but Closure doesn't include the TS sources properly at this time.
#
# `checkTypes` is blocked b/c TypeScript can perform our type checking... and it causes an error
# with TypeScript's `extend` implementation (it doesn't recognize a constructor without manual edits).
# We also get a global `this` warning from the same.
#
# `checkVars` is blocked b/c Closure will otherwise fail on TypeScript namespacing, as each original TS
# source file will redeclare the namespace variable, despite being merged into a single file post-compilation.
#
# `jsDocMissingType` prevents errors on type documentation Closure thinks is missing.  TypeScript may not
# have the same requirements, and we trust TypeScript over Closure.
minifier_warnings="--jscomp_error=* --jscomp_off=lintChecks --jscomp_off=unusedLocalVariables --jscomp_off=globalThis --jscomp_off=checkTypes --jscomp_off=checkVars --jscomp_off=jsdocMissingType --jscomp_off=uselessCode --jscomp_off=missingRequire --jscomp_off=strictMissingRequire"

# We use these to prevent Closure from auto-inserting its own polyfills.  Turns out, they can break in the
# WebView used by Android API 19, which our app still supported when written.  Unsure if it can still occur
# in Android API 21, our current minimum.
#
# Also, we currently apply all needed polyfills either manually or during TS compilation; we don't need the extra,
# excess code.
minifier_lang_specs="--language_in ECMASCRIPT5 --language_out ECMASCRIPT5"
minifycmd="$JAVA -jar $minifier --compilation_level WHITESPACE_ONLY $minifier_warnings --generate_exports $minifier_lang_specs"

readonly minifier
readonly minifycmd

minified_sourcemap_cleaner="build/tools/building/sourcemap-root/index.mjs"

# Fails the build if a specified file does not exist.
assert_exists ( ) {
  if ! [ -f $1 ]; then
    echo "Build failed:  expected file ${COLOR_GREY}$1${COLOR_RESET} is missing."
    exit 1
  fi
}

# $1 - base file name
# $2 - output path
# $3 - optimization level
# $4 - extra path info to add to minified sourcemap "sourceRoot" property.
# $5 - additional output wrapper
minify ( ) {
    if [ $# -ge 4 ]; then
        cleanerOptions="--clean --sourceRoot $4"
    else
        cleanerOptions="--clean"
    fi

    if [ $# -ge 5 ]; then
        wrapper=$5
    else
        wrapper="%output%"
    fi

    local INPUT="$1"
    local INPUT_FILE="$(basename $1)"
    local INPUT_DIR="$(dirname $1)"
    local INPUT_SOURCEMAP="$INPUT_DIR/$INPUT_FILE.map"
    local OUTPUT="$2"
    local OUTPUT_FILE="$(basename $2)"
    local OUTPUT_SOURCEMAP="$(dirname $2)/$OUTPUT_FILE.map"

    # --source_map_location_mapping - maps paths on INPUT source maps for consumption by Closure.
    # ../../.. => keymanapp, ../.. => keymanapp/keyman.  We have TS root sources on 'keyman'.
    $minifycmd --source_map_input "$INPUT|$INPUT_SOURCEMAP" \
        --create_source_map "$OUTPUT_SOURCEMAP" --source_map_include_content \
        --source_map_location_mapping "$INPUT_DIR|../../.." \
        --js "$INPUT" --compilation_level $3 \
        --js_output_file "$OUTPUT" --warning_level VERBOSE --output_wrapper "$wrapper
//# sourceMappingURL=$INPUT_FILE.map"

    # Now to clean the source map.
    assert_exists "$OUTPUT"
    assert_exists "$OUTPUT_SOURCEMAP"

    # "Clean" the minified output sourcemaps.
    node $minified_sourcemap_cleaner "$INPUT_SOURCEMAP" "$OUTPUT_SOURCEMAP" $cleanerOptions
}

# Copies specified engine resources to the specified target's build output directories.
#
# ### Parameters
#
# * 1: `product`    the product's source path under src/
# * 2: `outputs`    an array of resource types to copy over
#
# ### Example
#
# ```bash
#   copy_resources app/web osk ui
# ```
copy_resources ( ) {
  local COMPILE_TARGET=$1
  shift

  local RESOURCES_TO_COPY=("$@")

  # We leave out $INTERMEDIATE here, as it's not a 'release' of any sort and
  # thus doesn't need to publish sources or resources.
  local CONFIGS=($DEBUG)

  if ! builder_has_option --skip-minify; then
    CONFIGS+=($RELEASE)
  fi

  echo

  for CONFIG in "${CONFIGS[@]}";
  do
    local CONFIG_OUT_PATH="$(output_path $COMPILE_TARGET $CONFIG)"

    echo Copying resources to $CONFIG_OUT_PATH/src

    for RESOURCE in "${RESOURCES_TO_COPY[@]}";
    do
      mkdir -p "$CONFIG_OUT_PATH/$RESOURCE"
      mkdir -p "$CONFIG_OUT_PATH/src/resources/$RESOURCE"

      echo "- $SOURCE/resources/$RESOURCE/ => $CONFIG_OUT_PATH/$RESOURCE"
      cp -Rf "$SOURCE/resources/$RESOURCE"  "$CONFIG_OUT_PATH/"  >/dev/null
    done

    echo
  done
}

# Copies specified source folders corresponding to the specified target's build
# output directories.
#
# ### Parameters
#
# * 1: `product`    the product's source path under src/
# * 2: `outputs`    an array of src/ subfolders to copy over
#
# ### Example
#
# ```bash
#   copy_sources app/web engine resources/osk
# ```
copy_sources ( ) {
  local COMPILE_TARGET=$1
  shift

  local SOURCES_TO_COPY=("$@")

  # We leave out $INTERMEDIATE here, as it's not a 'release' of any sort and
  # thus doesn't need to publish sources or resources.
  CONFIGS=($DEBUG)

  if ! builder_has_option --skip-minify; then
    CONFIGS+=($RELEASE)
  fi

  for CONFIG in "${CONFIGS[@]}";
  do
    CONFIG_OUT_PATH="$(output_path $COMPILE_TARGET $CONFIG)"
    echo Copying $COMPILE_TARGET sources to $CONFIG_OUT_PATH/src

    rm -rf "$CONFIG_OUT_PATH/src"
    mkdir -p "$CONFIG_OUT_PATH/src"
    echo $VERSION_PATCH > "$CONFIG_OUT_PATH/src/version.txt"

    for SOURCE_FOLDER in "${SOURCES_TO_COPY[@]}";
    do
      echo "- $SOURCE/$SOURCE_FOLDER/ => $CONFIG_OUT_PATH/src/$SOURCE_FOLDER/"
      mkdir -p "$CONFIG_OUT_PATH/src/$SOURCE_FOLDER"
      cp -Rf  "$SOURCE/$SOURCE_FOLDER/"*    "$CONFIG_OUT_PATH/src/$SOURCE_FOLDER/"
    done

    echo
  done
}

# Compiles compiled scripts from the first folder specified to the second folder specified.
#
# ### Parameters
#
# * 1: `src`      the folder containing scripts to be copied
# * 2: `dst`      the destination folder for the copy operation
# * 3: `scripts`  an array of filenames for expected compiled scripts
#
# ### Example
#
# ```bash
#   compile_and_minify build/app/web/obj build/app/web/debug keymanweb.js
# ```
copy_outputs ( ) {
  local src="$1"
  local dst="$2"

  shift
  shift

  local BASE_SCRIPTS=("$@")

  mkdir -p "$dst"

  for SCRIPTJS in "${BASE_SCRIPTS[@]}";
  do
    cp -Rf "$src/$SCRIPTJS"             "$dst/"
    cp -Rf "$src/$SCRIPTJS.map"         "$dst/"
  done
}

# Compiles all build products corresponding to the specified target.
#
# ### Parameters
#
# * 1: `product`    the product's source path under src/
#
# ### Example
#
# ```bash
#   compile app/embed
# ```
compile ( ) {
  if [ $# -lt 1 ]; then
    fail "Scripting error: insufficient argument count!"
  fi

  local COMPILE_TARGET=$1
  local COMPILED_INTERMEDIATE_PATH="$(output_path $COMPILE_TARGET $INTERMEDIATE)"

  $compilecmd -b src/$COMPILE_TARGET -v

  echo $COMPILE_TARGET TypeScript compiled under $COMPILED_INTERMEDIATE_PATH
}

# Finalizes all build products corresponding to the specified target.
# This should be called after `compile` for all `app/` targets.
#
# ### Parameters
#
# * 1: `product`    the product's source path under src/
# * 2: `outputs`    an array of expected output script files for the build
#
# ### Example
#
# ```bash
#   compile app/embed
#   finalize app/embed keyman.js
# ```
finalize ( ) {
  if [ $# -lt 2 ]; then
    fail "Scripting error: insufficient argument count!"
  fi

  local COMPILE_TARGET=$1
  local COMPILED_INTERMEDIATE_PATH="$(output_path $COMPILE_TARGET $INTERMEDIATE)"
  local DEBUG_OUT_PATH="$(output_path $COMPILE_TARGET $DEBUG)"
  local RELEASE_OUT_PATH="$(output_path $COMPILE_TARGET $RELEASE)"

  shift

  local OUTPUT_SCRIPTS=("$@")

  # START:  debug output

  mkdir -p "$DEBUG_OUT_PATH"
  copy_outputs "$COMPILED_INTERMEDIATE_PATH" "$DEBUG_OUT_PATH" "${OUTPUT_SCRIPTS[@]}"

  echo Compiled $COMPILE_TARGET debug version saved under $DEBUG_OUT_PATH: ${OUTPUT_SCRIPTS[*]}

  # START:  release output
  if ! builder_has_option --skip-minify; then
    for SCRIPT in "${OUTPUT_SCRIPTS[@]}";
    do
      minify "$COMPILED_INTERMEDIATE_PATH/$SCRIPT" "$RELEASE_OUT_PATH/$SCRIPT" SIMPLE_OPTIMIZATIONS
    done

    echo Compiled $COMPILE_TARGET release version saved under $RELEASE_OUT_PATH: ${OUTPUT_SCRIPTS[*]}
  else
    # The prior 'release' is now outdated:  delete it.
    rm -rf "$RELEASE_OUT_PATH"
  fi
}

#### Build action definitions ####

if builder_start_action configure; then
  verify_npm_setup

  if ! builder_has_option --no-minify; then
    # NPM install is required for the file to be present.
    if ! [ -f $minifier ];
    then
      echo File $minifier does not exist:  have you set the environment variable \$CLOSURECOMPILERPATH?
      exit 1
    fi
  fi

  builder_finish_action success configure
fi

## Clean actions

# Possible issue:  there's no clear rule to `clean` the engine, which is auto-built
# by build:embed and build:web.
#
# Some sort of command to run ONLY for a general `clean` (no target specified) would
# be perfect for that, I think.

if builder_start_action clean:engine; then
  src/engine/build.sh clean
  builder_finish_action success clean:engine
fi

if builder_start_action clean:embed; then
  rm -rf "$(output_path $EMBEDDED)"
  builder_finish_action success clean:embed
fi

if builder_start_action clean:web; then
  rm -rf "$(output_path $WEB)"
  builder_finish_action success clean:web
fi

if builder_start_action clean:ui; then
  rm -rf "$(output_path $UI)"
  builder_finish_action success clean:ui
fi

if builder_start_action clean:samples; then
  rm -f $PREDICTIVE_TEXT_OUTPUT

  builder_finish_action success clean:samples
fi

if builder_start_action clean:tools; then
  src/tools/build.sh clean

  builder_finish_action success clean:tools
fi

## Build actions

# Adds a simple version 'header' when there's a main engine build product.
if builder_has_action build:embed || \
   builder_has_action build:web || \
   builder_has_action build:ui; then

  echo ""
  echo "${COLOR_PURPLE}Compiling version ${VERSION}${COLOR_RESET}"
fi

echo ""


if builder_start_action build:engine; then
  src/engine/build.sh build

  builder_finish_action success build:engine
fi

if builder_start_action build:embed; then
  compile $EMBEDDED
  finalize $EMBEDDED ${EMBED_TARGETS[@]}

  # The embedded version doesn't use UI modules.
  copy_resources $EMBEDDED osk
  copy_sources $EMBEDDED app/embed engine resources/osk

  builder_finish_action success build:embed

  # TODO:  handle this block somehow.

  # if [ $UPLOAD_EMBED_SENTRY = true ]; then  # upload-symbols:embed
  #   if [ $BUILD_DEBUG_EMBED = true ]; then
  #     ARTIFACT_FOLDER="release/unminified/embedded"
  #     pushd $EMBED_OUTPUT_NO_MINI
  #   else
  #     ARTIFACT_FOLDER="release/embedded"
  #     pushd $EMBED_OUTPUTs
  #   fi
  #   echo "Uploading to Sentry..."
  #   npm run sentry-cli -- releases files "$SENTRY_RELEASE_VERSION" upload-sourcemaps --strip-common-prefix $ARTIFACT_FOLDER --rewrite --ext js --ext map --ext ts || fail "Sentry upload failed."
  #   echo "Upload successful."
  #   popd
  # fi
fi

### -embed section complete.

if builder_start_action build:web; then
  compile $WEB
  finalize $WEB ${WEB_TARGETS[@]}

  # The testing pages need both osk & ui resources in the same place.
  copy_resources $WEB osk ui
  copy_sources $WEB app/web engine resources/osk

  builder_finish_action success build:web
fi

if builder_start_action build:ui; then
  compile $UI
  finalize $UI ${UI_TARGETS[@]}

  copy_resources $UI ui
  copy_sources $UI app/ui resources/ui

  builder_finish_action success build:ui
fi

if builder_start_action build:tools; then
  src/tools/build.sh
  builder_finish_action success build:tools
fi

if builder_start_action build:samples; then
  # Some test pages actually have build scripts.
  ./src/test/manual/embed/android-harness/build.sh  # is not yet builder-based.

  echo "Copying samples & test page resources..."
  # Should probably be changed into a build script for the `prediction-ui` test page.
  cp "${PREDICTIVE_TEXT_SOURCE}" "${PREDICTIVE_TEXT_OUTPUT}"

  # Which could then have a parallel script for `prediction-mtnt` that downloads + extracts
  # the current MTNT model.

  builder_finish_action success build:samples;
fi

if builder_start_action test:web; then
  if builder_has_option --all; then
    ./test.sh
  else
    ./test.sh :engine
  fi

  builder_finish_action success test:web
fi

# TODO:  handle the block below somehow.

# # We can only upload 'web' / 'native' artifacts after ALL are done compiling.
# if [ $UPLOAD_WEB_SENTRY = true ]; then
#     pushd $WEB_OUTPUT
#     echo "Uploading to Sentry..."
#     npm run sentry-cli -- releases files "$SENTRY_RELEASE_VERSION" upload-sourcemaps --strip-common-prefix release/web/ --rewrite --ext js --ext map --ext ts || fail "Sentry upload failed."
#     echo "Upload successful."
#     popd
# fi