From 339cff11cf47fae0069e992e06a63533af213834 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 13 Mar 2023 06:15:43 +0700 Subject: [PATCH 01/12] chore(common): support shorthand for build.sh Fixes #8396 --- resources/builder.inc.sh | 167 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 155 insertions(+), 12 deletions(-) diff --git a/resources/builder.inc.sh b/resources/builder.inc.sh index eb797606e1..85a5b0ee5d 100755 --- a/resources/builder.inc.sh +++ b/resources/builder.inc.sh @@ -235,6 +235,67 @@ _builder_item_in_glob_array() { return 1 } +# +# Expands a shorthand item into a full match from an array of possibilities; +# reports an error if there are ambiguous options. Note that this function +# returns the number of matches, so 0 = no match, 1 = a precise match. +# +# ### Parameters +# +# * 1: `item` item to search for in array, e.g. "t" +# * 2: `array` bash array, e.g. `array=(one two three)` +# +# ### Description +# +# Does a substring search by regex for +# +# ### Example +# +# ```bash +# actions=(clean configure build test) +# +# action=`_builder_expand_shorthand $1 "${actions[@]}"` && +# builder_die "Unrecognized parameter $1" || +# case $? in +# 1) echo "Parameter $1 matches {$action}" +# ;; +# *) builder_die "Parameter $1 has $? matches, could mean any of {$action}" +# esac +# ``` +# +_builder_expand_shorthand() { + local item=$1 + shift + local count=0 + local result= + for e; do + if [[ $e == $item ]]; then + # Exact match trumps substring matches + echo $item + return 1 + fi + if [[ $e == "$item"* ]]; then + count=$((count+1)) + if [[ $count == 2 ]]; then + printf "$result" + printf ", $e" + result=$item + elif [[ $count -gt 2 ]]; then + printf ", $e" + else + result=$e + fi + fi + done + + if [[ $count -lt 2 ]]; then + echo $result + else + echo + fi + return $count +} + _builder_item_is_target() { local item="$1" @@ -1015,19 +1076,72 @@ builder_describe_internal_dependency() { # Parameters # 1: $@ command-line arguments builder_parse() { - _builder_record_function_call builder_parse - local _builder_params="$@" + local exp=() + builder_extra_params=() + while [[ $# -gt 0 ]] ; do + local action= target= + local key="$1" + if [[ $key == "--" ]]; then + shift + builder_extra_params=("$@") + break + fi + + if [[ $key =~ ^- ]]; then + exp+=($key) + else + # Expand comma separated values + if [[ $key =~ : ]]; then + IFS=: read -r action target <<< $key + else + action="$key" + target= + fi + + local actions targets + IFS=, read -r -a actions <<< "$action" + IFS=, read -r -a targets <<< "$target" + + if [[ "${#actions[@]}" -eq 0 ]]; then + # No actions, so must be at least one :target + for target in "${targets[@]}"; do + exp+=(:$target) + done + else + for action in "${actions[@]}"; do + if [[ "${#targets[@]}" -eq 0 ]]; then + # No :targets so just expand actions + exp+=($action) + else + # Actions:targets, expand them all + for target in "${targets[@]}"; do + exp+=($action:$target) + done + fi + done + fi + fi + + shift + done + + _builder_parse_expanded_parameters "${exp[@]}" +} + +_builder_parse_expanded_parameters() { _builder_build_deps=--deps builder_verbose= builder_debug= - builder_extra_params=() + local _params=($@) _builder_chosen_action_targets=() _builder_chosen_options=() _builder_current_action= + local n=0 + # Process command-line arguments while [[ $# -gt 0 ]] ; do local key="$1" @@ -1035,12 +1149,6 @@ builder_parse() { local target= local e has_action has_target has_option longhand_option - if [[ $key == "--" ]]; then - shift - builder_extra_params=("$@") - break - fi - if [[ $key =~ : ]]; then IFS=: read -r action target <<< $key target=:$target @@ -1049,9 +1157,39 @@ builder_parse() { target= fi + # Expand shorthand parameters + + new_action=`_builder_expand_shorthand $action "${_builder_actions[@]}"` || + case $? in + 1) + action=$new_action + ;; + *) + builder_warn "Parameter $action has $? matches, could mean any of {$new_action}" + exit 1 + ;; + esac + + new_target=`_builder_expand_shorthand $target "${_builder_targets[@]}"` || + case $? in + 1) + target=$new_target + ;; + *) + builder_warn "Parameter $target has $? matches, could mean any of {$new_target}" + exit 1 + ;; + esac + _builder_item_in_array "$action" "${_builder_actions[@]}" && has_action=1 || has_action=0 _builder_item_in_array "$target" "${_builder_targets[@]}" && has_target=1 || has_target=0 + if (( has_action )) || (( has_target )); then + # Document parameter expansion for end use + _params[$n]=$action$target + fi + n=$((n + 1)) + # Expand short -o to --option in options lookup if [[ ! -z ${_builder_options_short[$key]+x} ]]; then key=${_builder_options_short[$key]} @@ -1060,7 +1198,7 @@ builder_parse() { if (( has_action )) && (( has_target )); then # apply the selected action and selected target - _builder_chosen_action_targets+=("$key") + _builder_chosen_action_targets+=("$action$target") elif (( has_action )); then # apply the selected action to all targets if [[ ! -z $target ]]; then @@ -1163,7 +1301,7 @@ builder_parse() { if builder_is_dep_build; then builder_echo setmark "dependency build, started by $builder_dep_parent" - builder_echo grey "build.sh parameters: <${_builder_params[@]}>" + builder_echo grey "build.sh parameters: <${_params[@]}>" if [[ -z ${_builder_deps_built+x} ]]; then builder_die "FATAL ERROR: Expected --builder-deps-built parameter" fi @@ -1171,7 +1309,12 @@ builder_parse() { # This is a top-level invocation, not a dependency build, so we want to # track which dependencies have been built, so they don't get built multiple # times. - builder_echo setmark "build.sh parameters: <${_builder_params[@]}>" + # TODO: consider printing expanded builder parameters instead of shorthand + echo "${_params[@]}" + builder_echo setmark "build.sh parameters: <${_params[@]}>" + if [[ ${#builder_extra_params[@]} -gt 0 ]]; then + builder_echo grey "build.sh extar parameters: <${builder_extra_params[@]}>" + fi _builder_deps_built=`mktemp` fi From 22930be045ff90150466b917996501e0330c7c0d Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 13 Mar 2023 06:20:28 +0700 Subject: [PATCH 02/12] chore(common): minor cleanup --- resources/builder.inc.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/resources/builder.inc.sh b/resources/builder.inc.sh index 85a5b0ee5d..8c7e06cf8a 100755 --- a/resources/builder.inc.sh +++ b/resources/builder.inc.sh @@ -152,10 +152,9 @@ builder_echo() { local color=white message= mark= if [[ $# -gt 1 ]]; then color="$1" - message="$2" - else - message="$1" + shift fi + message="$*" if [[ ! -z ${COLOR_RED+x} ]]; then case $color in @@ -1159,7 +1158,7 @@ _builder_parse_expanded_parameters() { # Expand shorthand parameters - new_action=`_builder_expand_shorthand $action "${_builder_actions[@]}"` || + new_action=$(_builder_expand_shorthand $action "${_builder_actions[@]}") || case $? in 1) action=$new_action @@ -1170,7 +1169,7 @@ _builder_parse_expanded_parameters() { ;; esac - new_target=`_builder_expand_shorthand $target "${_builder_targets[@]}"` || + new_target=$(_builder_expand_shorthand $target "${_builder_targets[@]}") || case $? in 1) target=$new_target From 92ed99c4d660c27e91661f89b42c6196134e1b29 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 13 Mar 2023 07:18:09 +0700 Subject: [PATCH 03/12] chore(common): build script performance improvements Improves build script performance by: * using built-ins wherever possible (e.g. string splitting) * eliminating redundant code * using absolute (to $KEYMAN_ROOT) rather than relative paths to avoid realpath * removing unnecessary `npm run` calls BEFORE | AFTER -------------------------------|---------------------- time ./core/build.sh --help | real 0m2.116s | real 0m0.874s user 0m0.578s | user 0m0.198s sys 0m0.984s | sys 0m0.289s -------------------------------|---------------------- time ./web/build.sh --help | real 0m3.523s | real 0m0.757s user 0m1.166s | user 0m0.320s sys 0m2.273s | sys 0m0.455s -------------------------------|---------------------- time ./web/build.sh -d | real 1m34.750s | real 0m59.974s user 0m9.721s | user 0m6.284s sys 0m19.652s | sys 0m13.202s @keymanapp-test-bot skip --- common/predictive-text/build.sh | 8 +-- common/web/input-processor/build.sh | 14 ++-- common/web/keyboard-processor/build.sh | 14 ++-- common/web/keyman-version/build.sh | 5 +- common/web/lm-worker/build.sh | 6 +- common/web/recorder/build.sh | 14 ++-- common/web/sentry-manager/src/build.sh | 2 +- common/web/types/build.sh | 4 +- common/web/utils/build.sh | 6 +- core/build.sh | 28 ++++---- core/commands.inc.sh | 1 - developer/src/kmc-keyboard/build.sh | 2 +- developer/src/kmc-model-info/build.sh | 4 +- developer/src/kmc-model/build.sh | 2 +- developer/src/kmc-package/build.sh | 2 +- developer/src/kmc/build.sh | 10 +-- resources/build/build-utils.sh | 35 ++++++---- resources/builder.inc.sh | 67 ++++++++++--------- web/build.sh | 36 +++++----- web/src/engine/build.sh | 17 +++-- web/src/tools/build.sh | 4 +- .../tools/building/sourcemap-root/build.sh | 6 +- web/src/tools/testing/bulk_rendering/build.sh | 2 +- web/src/tools/testing/recorder/build.sh | 10 +-- 24 files changed, 159 insertions(+), 140 deletions(-) diff --git a/common/predictive-text/build.sh b/common/predictive-text/build.sh index 64377629b4..617eaa379b 100755 --- a/common/predictive-text/build.sh +++ b/common/predictive-text/build.sh @@ -28,8 +28,8 @@ cd "$(dirname "$THIS_SCRIPT")" # "@../models/wordbreakers" builder_describe "Builds the lm-layer module" \ - "@../web/keyman-version" \ - "@../web/lm-worker" \ + "@/common/web/keyman-version" \ + "@/common/web/lm-worker" \ "clean" \ "configure" \ "build" \ @@ -42,8 +42,8 @@ builder_describe_outputs \ configure /node_modules \ configure:headless /node_modules \ configure:browser /node_modules \ - build:headless build/headless.js \ - build:browser build/index.js + build:headless /common/predictive-text/build/headless.js \ + build:browser /common/predictive-text/build/index.js builder_parse "$@" diff --git a/common/web/input-processor/build.sh b/common/web/input-processor/build.sh index 03e012aa19..c7995e5d42 100755 --- a/common/web/input-processor/build.sh +++ b/common/web/input-processor/build.sh @@ -20,9 +20,9 @@ cd "$(dirname "$THIS_SCRIPT")" # TODO: remove :tools once kmlmc is a dependency for test:module builder_describe "Builds the standalone, headless form of Keyman Engine for Web's input-processor module" \ - "@../keyman-version" \ - "@../keyboard-processor" \ - "@../../predictive-text" \ + "@/common/web/keyman-version" \ + "@/common/web/keyboard-processor" \ + "@/common/predictive-text" \ "clean" \ "configure" \ "build" \ @@ -35,7 +35,7 @@ builder_describe_outputs \ configure /node_modules \ configure:module /node_modules \ configure:tools /node_modules \ - build:module build/index.js \ + build:module /common/web/input-processor/build/index.js \ build:tools /developer/src/kmc/build/src/kmlmc.js # TODO: remove this once kmlmc is a dependency builder_parse "$@" @@ -70,7 +70,7 @@ if builder_start_action build:tools; then fi if builder_start_action build:module; then - npm run tsc -- -b src/tsconfig.json + tsc -b src/tsconfig.json builder_finish_action success build:module fi @@ -83,9 +83,9 @@ if builder_start_action test:module; then fi # Build the leaf-style, bundled version of input-processor for use in testing. - npm run tsc -- -b src/tsconfig.bundled.json + tsc -b src/tsconfig.bundled.json - npm run mocha -- --recursive $FLAGS ./tests/cases/ + mocha --recursive $FLAGS ./tests/cases/ builder_finish_action success test:module fi \ No newline at end of file diff --git a/common/web/keyboard-processor/build.sh b/common/web/keyboard-processor/build.sh index 6e5b3bf4fe..ebcab580ab 100755 --- a/common/web/keyboard-processor/build.sh +++ b/common/web/keyboard-processor/build.sh @@ -19,9 +19,9 @@ cd "$THIS_SCRIPT_PATH" builder_describe \ "Compiles the web-oriented utility function module." \ - "@../recorder test" \ - "@../keyman-version" \ - "@../utils" \ + "@/common/web/recorder test" \ + "@/common/web/keyman-version" \ + "@/common/web/utils" \ configure \ clean \ build \ @@ -30,7 +30,7 @@ builder_describe \ builder_describe_outputs \ configure /node_modules \ - build build/index.js + build /common/web/keyboard-processor/build/index.js builder_parse "$@" @@ -45,12 +45,12 @@ if builder_start_action clean; then fi if builder_start_action build; then - npm run tsc -- --build "$THIS_SCRIPT_PATH/src/tsconfig.json" + tsc --build "$THIS_SCRIPT_PATH/src/tsconfig.json" builder_finish_action success build fi if builder_start_action test; then - npm run tsc -- --build "$THIS_SCRIPT_PATH/src/tsconfig.bundled.json" + tsc --build "$THIS_SCRIPT_PATH/src/tsconfig.bundled.json" builder_heading "Running Keyboard Processor test suite" @@ -60,7 +60,7 @@ if builder_start_action test; then FLAGS="$FLAGS --reporter mocha-teamcity-reporter" fi - npm run mocha -- --recursive $FLAGS ./tests/cases/ + mocha --recursive $FLAGS ./tests/cases/ builder_finish_action success test fi \ No newline at end of file diff --git a/common/web/keyman-version/build.sh b/common/web/keyman-version/build.sh index c89fa1847a..b7735967c7 100755 --- a/common/web/keyman-version/build.sh +++ b/common/web/keyman-version/build.sh @@ -28,7 +28,7 @@ builder_describe "Build the include script for current Keyman version" \ builder_describe_outputs \ configure "/node_modules" \ - build "build/keyman-version.mjs" + build "/common/web/keyman-version/build/keyman-version.mjs" builder_parse "$@" @@ -85,7 +85,8 @@ if builder_start_action build; then if builder_is_dep_build; then builder_echo "skipping tsc -b; will be completed by $builder_dep_parent" else - npm run build -- $builder_verbose + echo 'Building @keymanapp/keyman-version' + tsc -b $builder_verbose fi builder_finish_action success build diff --git a/common/web/lm-worker/build.sh b/common/web/lm-worker/build.sh index f4d38afc10..cc6aaf5162 100755 --- a/common/web/lm-worker/build.sh +++ b/common/web/lm-worker/build.sh @@ -78,12 +78,12 @@ wrap-worker-code ( ) { builder_describe \ "Compiles the Language Modeling Layer for common use in predictive text and autocorrective applications." \ - "@../keyman-version" \ + "@/common/web/keyman-version" \ configure clean build test builder_describe_outputs \ configure /node_modules \ - build build/index.js + build /common/web/lm-worker/build/index.js builder_parse "$@" @@ -113,7 +113,7 @@ if builder_start_action build; then fi # Build worker with tsc first - npm run build -- $builder_verbose || builder_die "Could not build worker." + tsb -b $builder_verbose || builder_die "Could not build worker." # Wrap the worker code and create embedded index.js. Must be run after the # worker is built diff --git a/common/web/recorder/build.sh b/common/web/recorder/build.sh index 65edd2af9c..b9e578e054 100755 --- a/common/web/recorder/build.sh +++ b/common/web/recorder/build.sh @@ -19,7 +19,7 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" builder_describe \ "Compiles the web-oriented utility function module." \ - "@../keyman-version" \ + "@/common/web/keyman-version" \ configure \ clean \ build \ @@ -30,8 +30,8 @@ builder_describe_outputs \ configure "/node_modules" \ configure:module "/node_modules" \ configure:proctor "/node_modules" \ - build:module "build/index.js" \ - build:proctor "build/nodeProctor/index.js" + build:module "/common/web/recorder/build/index.js" \ + build:proctor "/common/web/recorder/build/nodeProctor/index.js" builder_parse "$@" @@ -41,21 +41,21 @@ if builder_start_action configure; then fi if builder_start_action clean:module; then - npm run tsc -- -b --clean "$THIS_SCRIPT_PATH/src/tsconfig.json" + tsc -b --clean "$THIS_SCRIPT_PATH/src/tsconfig.json" builder_finish_action success clean:module fi if builder_start_action clean:proctor; then - npm run tsc -- -b --clean "$THIS_SCRIPT_PATH/src/nodeProctor.tsconfig.json" + tsc -b --clean "$THIS_SCRIPT_PATH/src/nodeProctor.tsconfig.json" builder_finish_action success clean:proctor fi if builder_start_action build:module; then - npm run tsc -- --build "$THIS_SCRIPT_PATH/src/tsconfig.json" + tsc --build "$THIS_SCRIPT_PATH/src/tsconfig.json" builder_finish_action success build:module fi if builder_start_action build:proctor; then - npm run tsc -- --build "$THIS_SCRIPT_PATH/src/nodeProctor.tsconfig.json" + tsc --build "$THIS_SCRIPT_PATH/src/nodeProctor.tsconfig.json" builder_finish_action success build:proctor fi \ No newline at end of file diff --git a/common/web/sentry-manager/src/build.sh b/common/web/sentry-manager/src/build.sh index fe5c6bd434..7f3775702d 100755 --- a/common/web/sentry-manager/src/build.sh +++ b/common/web/sentry-manager/src/build.sh @@ -47,7 +47,7 @@ if [ $FETCH_DEPS = true ]; then "$KEYMAN_ROOT/common/web/keyman-version/build.sh" || builder_die "Could not build keyman-version" fi -npm run tsc -- --build "$THIS_SCRIPT_PATH/tsconfig.json" +tsc --build "$THIS_SCRIPT_PATH/tsconfig.json" if [ $? -ne 0 ]; then builder_die "Compilation of package for Sentry integration with KeymanWeb failed." fi \ No newline at end of file diff --git a/common/web/types/build.sh b/common/web/types/build.sh index dc376f4a0d..3c7e69b216 100755 --- a/common/web/types/build.sh +++ b/common/web/types/build.sh @@ -17,7 +17,7 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" cd "$THIS_SCRIPT_PATH" builder_describe "Build Keyman common file types module" \ - "@../keyman-version" \ + "@/common/web/keyman-version" \ "configure" \ "build" \ "clean" \ @@ -26,7 +26,7 @@ builder_describe "Build Keyman common file types module" \ "--dry-run,-n don't actually publish, just dry run" builder_describe_outputs \ configure /node_modules \ - build build/src/main.js + build /common/web/types/build/src/main.js builder_parse "$@" diff --git a/common/web/utils/build.sh b/common/web/utils/build.sh index 6c2ebbe9c9..9c31429a62 100755 --- a/common/web/utils/build.sh +++ b/common/web/utils/build.sh @@ -18,12 +18,12 @@ cd "$THIS_SCRIPT_PATH" builder_describe \ "Compiles the web-oriented utility function module." \ - "@../keyman-version" \ + "@/common/web/keyman-version" \ configure clean build builder_describe_outputs \ configure "/node_modules" \ - build "build/index.js" + build "/common/web/utils/build/index.js" builder_parse "$@" @@ -42,7 +42,7 @@ if builder_start_action build; then if builder_is_dep_build; then builder_echo "skipping tsc -b; will be completed by $builder_dep_parent" else - npm run tsc -- --build "$THIS_SCRIPT_PATH/tsconfig.json" + tsc --build "$THIS_SCRIPT_PATH/tsconfig.json" fi builder_finish_action success build fi \ No newline at end of file diff --git a/core/build.sh b/core/build.sh index 23ad94d283..caaeafb929 100755 --- a/core/build.sh +++ b/core/build.sh @@ -106,20 +106,20 @@ builder_describe_internal_dependency \ build:mac build:mac-arm64 builder_describe_outputs \ - configure:x86 build/x86/$CONFIGURATION/build.ninja \ - configure:x64 build/x64/$CONFIGURATION/build.ninja \ - configure:mac build/mac/$CONFIGURATION/ \ - configure:mac-x86_64 build/mac-x86_64/$CONFIGURATION/build.ninja \ - configure:mac-arm64 build/mac-arm64/$CONFIGURATION/build.ninja \ - configure:arch build/arch/$CONFIGURATION/build.ninja \ - configure:wasm build/wasm/$CONFIGURATION/build.ninja \ - build:x86 build/x86/$CONFIGURATION/src/libkmnkbp0.a \ - build:x64 build/x64/$CONFIGURATION/src/libkmnkbp0.a \ - build:mac build/mac/$CONFIGURATION/libkmnkbp0.a \ - build:mac-x86_64 build/mac-x86_64/$CONFIGURATION/src/libkmnkbp0.a \ - build:mac-arm64 build/mac-arm64/$CONFIGURATION/src/libkmnkbp0.a \ - build:arch build/arch/$CONFIGURATION/src/libkmnkbp0.a \ - build:wasm build/wasm/$CONFIGURATION/src/libkmnkbp0.a + configure:x86 /core/build/x86/$CONFIGURATION/build.ninja \ + configure:x64 /core/build/x64/$CONFIGURATION/build.ninja \ + configure:mac /core/build/mac/$CONFIGURATION/ \ + configure:mac-x86_64 /core/build/mac-x86_64/$CONFIGURATION/build.ninja \ + configure:mac-arm64 /core/build/mac-arm64/$CONFIGURATION/build.ninja \ + configure:arch /core/build/arch/$CONFIGURATION/build.ninja \ + configure:wasm /core/build/wasm/$CONFIGURATION/build.ninja \ + build:x86 /core/build/x86/$CONFIGURATION/src/libkmnkbp0.a \ + build:x64 /core/build/x64/$CONFIGURATION/src/libkmnkbp0.a \ + build:mac /core/build/mac/$CONFIGURATION/libkmnkbp0.a \ + build:mac-x86_64 /core/build/mac-x86_64/$CONFIGURATION/src/libkmnkbp0.a \ + build:mac-arm64 /core/build/mac-arm64/$CONFIGURATION/src/libkmnkbp0.a \ + build:arch /core/build/arch/$CONFIGURATION/src/libkmnkbp0.a \ + build:wasm /core/build/wasm/$CONFIGURATION/src/libkmnkbp0.a # Iterate through all possible targets; note that targets that cannot be built # on the current platform have already been excluded through the archtargets diff --git a/core/commands.inc.sh b/core/commands.inc.sh index 4a696c8555..f3eda70e6d 100644 --- a/core/commands.inc.sh +++ b/core/commands.inc.sh @@ -168,7 +168,6 @@ cleanup_visual_studio_path() { _new_path="$_new_path:$p" fi done - echo PATH="${_new_path:1}" unset VSINSTALLDIR } diff --git a/developer/src/kmc-keyboard/build.sh b/developer/src/kmc-keyboard/build.sh index 04280fa49b..e6ac49b35d 100755 --- a/developer/src/kmc-keyboard/build.sh +++ b/developer/src/kmc-keyboard/build.sh @@ -28,7 +28,7 @@ builder_describe "Build Keyman kmc Keyboard Compiler module" \ "--dry-run,-n don't actually publish, just dry run" builder_describe_outputs \ configure /node_modules \ - build build/src/main.js + build /developer/src/kmc-keyboard/build/src/main.js builder_parse "$@" diff --git a/developer/src/kmc-model-info/build.sh b/developer/src/kmc-model-info/build.sh index 1a877cb854..da212a417b 100755 --- a/developer/src/kmc-model-info/build.sh +++ b/developer/src/kmc-model-info/build.sh @@ -17,7 +17,7 @@ cd "$THIS_SCRIPT_PATH" . "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" builder_describe "Build Keyman kmc Lexical Model model-info Compiler module" \ - "@../kmc-package" \ + "@/developer/src/kmc-package" \ "configure" \ "build" \ "clean" \ @@ -26,7 +26,7 @@ builder_describe "Build Keyman kmc Lexical Model model-info Compiler module" \ "--dry-run,-n don't actually publish, just dry run" builder_describe_outputs \ configure /node_modules \ - build build/src/model-info-compiler.js + build /developer/src/kmc-model-info/build/src/model-info-compiler.js builder_parse "$@" diff --git a/developer/src/kmc-model/build.sh b/developer/src/kmc-model/build.sh index 51aef36cea..2582c10451 100755 --- a/developer/src/kmc-model/build.sh +++ b/developer/src/kmc-model/build.sh @@ -28,7 +28,7 @@ builder_describe "Build Keyman kmc Lexical Model Compiler module" \ "--dry-run,-n don't actually publish, just dry run" builder_describe_outputs \ configure /node_modules \ - build build/src/main.js + build /developer/src/kmc-model/build/src/main.js builder_parse "$@" diff --git a/developer/src/kmc-package/build.sh b/developer/src/kmc-package/build.sh index ae69a60c6b..c17363729c 100755 --- a/developer/src/kmc-package/build.sh +++ b/developer/src/kmc-package/build.sh @@ -26,7 +26,7 @@ builder_describe "Build Keyman kmc Package Compiler module" \ "--dry-run,-n don't actually publish, just dry run" builder_describe_outputs \ configure /node_modules \ - build build/src/kmp-compiler.js + build /developer/src/kmc-package/build/src/kmp-compiler.js builder_parse "$@" #------------------------------------------------------------------------------------------------------------------- diff --git a/developer/src/kmc/build.sh b/developer/src/kmc/build.sh index f85302f25e..fd14e00e4c 100755 --- a/developer/src/kmc/build.sh +++ b/developer/src/kmc/build.sh @@ -19,10 +19,10 @@ cd "$THIS_SCRIPT_PATH" builder_describe "Build Keyman Keyboard Compiler kmc" \ "@/common/web/keyman-version" \ "@/common/web/types" \ - "@../kmc-keyboard" \ - "@../kmc-model" \ - "@../kmc-model-info" \ - "@../kmc-package" \ + "@/developer/src/kmc-keyboard" \ + "@/developer/src/kmc-model" \ + "@/developer/src/kmc-model-info" \ + "@/developer/src/kmc-package" \ "configure runs 'npm ci' on root folder" \ "build (default) builds kmc to build/" \ "clean cleans build/ folder" \ @@ -33,7 +33,7 @@ builder_describe "Build Keyman Keyboard Compiler kmc" \ "--dry-run,-n don't actually publish, just dry run" builder_describe_outputs \ configure /node_modules \ - build build/src/kmc.js + build /developer/src/kmc/build/src/kmc.js builder_parse "$@" diff --git a/resources/build/build-utils.sh b/resources/build/build-utils.sh index 8c497b06b0..a763548fab 100755 --- a/resources/build/build-utils.sh +++ b/resources/build/build-utils.sh @@ -35,17 +35,17 @@ SHLVL=0 function findKeymanRoot() { - # See https://stackoverflow.com/questions/59895/how-to-get-the-source-directory-of-a-bash-script-from-within-the-script-itself - # None of the answers are 100% correct for cross-platform - # On macOS, requires coreutils (`brew install coreutils`) - local SCRIPT=$(readlink -f "${BASH_SOURCE[0]}") - KEYMAN_ROOT="${SCRIPT%/*/*/*}" - readonly KEYMAN_ROOT + # We don't need readlink here because our standard script prolog does a + # readlink -f already so we will have already escaped from any symlinks + # But we still need to canonicalize paths to remove ../../.. + KEYMAN_ROOT="${BASH_SOURCE[0]%/*/*/*}" + KEYMAN_ROOT="$( cd "$KEYMAN_ROOT" && echo "$PWD" )" + readonly KEYMAN_ROOT } function findVersion() { local VERSION_MD="$KEYMAN_ROOT/VERSION.md" - VERSION=`cat $VERSION_MD | tr -d "[:space:]"` + VERSION=$(trim $(<"$VERSION_MD")) [[ "$VERSION" =~ ^([[:digit:]]+)\.([[:digit:]]+)\.([[:digit:]]+)$ ]] && { VERSION_MAJOR="${BASH_REMATCH[1]}" VERSION_MINOR="${BASH_REMATCH[2]}" @@ -118,13 +118,22 @@ function findVersion() { export VERSION_GIT_TAG } +function trim() { + local var="$*" + # remove leading whitespace characters + var="${var#"${var%%[![:space:]]*}"}" + # remove trailing whitespace characters + var="${var%"${var##*[![:space:]]}"}" + printf '%s' "$var" +} + function findTier() { - local TIER_MD="$KEYMAN_ROOT/TIER.md" - TIER=`cat $TIER_MD | tr -d "[:space:]"` - [[ "$TIER" =~ ^(alpha|beta|stable)$ ]] || { - echo "Invalid TIER.md file: expected alpha, beta or stable." - exit 1; - } + local TIER_MD="$KEYMAN_ROOT/TIER.md" + TIER=$(trim $(<"$TIER_MD")) + [[ "$TIER" =~ ^(alpha|beta|stable)$ ]] || { + echo "Invalid TIER.md file: expected alpha, beta or stable." + exit 1; + } } function printBuildNumberForTeamCity() { diff --git a/resources/builder.inc.sh b/resources/builder.inc.sh index 8c7e06cf8a..2d484b02f3 100755 --- a/resources/builder.inc.sh +++ b/resources/builder.inc.sh @@ -27,11 +27,9 @@ function _builder_init() { } function _builder_findRepoRoot() { - # See https://stackoverflow.com/questions/59895/how-to-get-the-source-directory-of-a-bash-script-from-within-the-script-itself - # None of the answers are 100% correct for cross-platform - # On macOS, requires coreutils (`brew install coreutils`) - local SCRIPT=$(readlink -f "${BASH_SOURCE[0]}") - REPO_ROOT="${SCRIPT%/*/*}" + # We don't need readlink here because our standard script prolog does a + # readlink -f already so we will have already escaped from any symlinks + REPO_ROOT="${BASH_SOURCE[0]%/*/*}" readonly REPO_ROOT } @@ -77,20 +75,20 @@ function _builder_setBuildScriptIdentifiers() { # 1: use_color true or false builder_use_color() { if $1; then - COLOR_RED=$(tput setaf 1) - COLOR_GREEN=$(tput setaf 2) - COLOR_YELLOW=$(tput setaf 3) - COLOR_BLUE=$(tput setaf 4) - COLOR_PURPLE=$(tput setaf 5) - COLOR_TEAL=$(tput setaf 6) - COLOR_WHITE=$(tput setaf 252) - COLOR_BRIGHT_WHITE=$(tput setaf 255) - COLOR_GREY=$(tput setaf 8) - COLOR_RESET=$(tput sgr0) + # Using esc codes instead of tput for performance + COLOR_RED='\x1b[31m' # $(tput setaf 1) + COLOR_GREEN='\x1b[32m' # $(tput setaf 2) + COLOR_YELLOW='\x1b[33m' # $(tput setaf 3) + COLOR_BLUE='\x1b[34m' # $(tput setaf 4) + COLOR_PURPLE='\x1b[35m' # $(tput setaf 5) + COLOR_TEAL='\x1b[36m' # $(tput setaf 6) + COLOR_WHITE='\x1b[38;5;252m' # $(tput setaf 252) + COLOR_BRIGHT_WHITE='\x1b[38;5;255m' # $(tput setaf 255) + COLOR_GREY='\x1b[90m' # $(tput setaf 8) + COLOR_RESET='\x1b(B\x1b[m' # $(tput sgr0) # e.g. VSCode https://code.visualstudio.com/updates/v1_69#_setmark-sequence-support - BUILDER_BOLD=$(tput bold) + BUILDER_BOLD='\x1b[1m' # $(tput bold) HEADING_SETMARK='\x1b]1337;SetMark\x07' - # Used by `builder_display_usage` when marking special terms (actions, targets, options) # in the plain-text description area. BUILDER_TERM_START="$COLOR_BLUE" @@ -619,8 +617,8 @@ _builder_expand_relative_path() { _builder_expand_action_target() { local input="$1" target= action= if [[ "$input" =~ : ]]; then - action=$(echo "$input" | cut -d: -f 1 -) - target=$(echo "$input" | cut -d: -f 2 -) + IFS=":" read -r action target <<< "$value" + target=":$target" else action=$input fi @@ -738,13 +736,16 @@ builder_describe() { declare -A -g _builder_target_paths # array of target child project paths declare -A -g _builder_dep_targets # array of :targets given for a specific dependency (comma separated if more than one) shift + local sub=() # describe each target, action, and option possibility while [[ $# -gt 0 ]]; do local key="$1" - local value="$(echo "$key" | cut -d" " -f 1 -)" + local value="$key" local description= if [[ $key =~ [[:space:]] ]]; then - description="$(_builder_trim "$(echo "$key" | cut -d" " -f 2- -)")" + IFS=" " read -r -a sub <<< "$key" + value="${sub[0]}" + description="$(_builder_trim "${sub[@]:1}")" fi if [[ $value =~ ^: ]]; then @@ -752,8 +753,9 @@ builder_describe() { local target_path= if [[ $value =~ = ]]; then # The target has a custom child project path - target_path="$(echo "$value" | cut -d= -f 2 -)" - value="$(echo "$value" | cut -d= -f 1 -)" + IFS="=" read -r -a sub <<< "$value" + target_path="${sub[@]:1}" + value="${sub[0]}" if [[ ! -d "$THIS_SCRIPT_PATH/$target_path" ]]; then builder_die "Target path '$target_path' for $value does not exist." fi @@ -773,8 +775,9 @@ builder_describe() { local dependency="${value:1}" local dependency_target= # all targets if [[ $dependency =~ : ]]; then - dependency_target=":$(echo "$dependency" | cut -d: -f 2 -)" - dependency="$(echo "$dependency" | cut -d: -f 1 -)" + IFS=":" read -r -a sub <<< "$dependency" + dependency_target=":${sub[@]:1}" + dependency="${sub[0]}" fi dependency="`_builder_expand_relative_path "$dependency"`" @@ -789,8 +792,9 @@ builder_describe() { # Look for a shorthand version of the option local option_var= if [[ $value =~ = ]]; then - option_var="$(echo "$value" | cut -d= -f 2 -)" - value="$(echo "$value" | cut -d= -f 1 -)" + IFS="=" read -r -a sub <<< "$value" + option_var="${sub[@]:1}" + value="${sub[0]}" fi local is_inheritable=false @@ -802,8 +806,9 @@ builder_describe() { fi if [[ $value =~ , ]]; then - local option_long="$(echo "$value" | cut -d, -f 1 -)" - local option_short="$(echo "$value" | cut -d, -f 2 -)" + IFS="," read -r -a sub <<< "$value" + local option_long="${sub[0]}" + local option_short="${sub[@]:1}" _builder_options+=($option_long) if $is_inheritable; then _builder_options_inheritable+=($option_long) @@ -888,8 +893,8 @@ function builder_describe_outputs() { path="`_builder_expand_relative_path "$path"`" if [[ $key =~ : ]]; then - action="$(echo "$key" | cut -d: -f 1 -)" - target=":$(echo "$key" | cut -d: -f 2 -)" + IFS=":" read -r -a action target <<< "$key" + target=":$target" else # Add dependency expected output file for all targets, as well as a # wildcard target match diff --git a/web/build.sh b/web/build.sh index 890f43fea2..0b534e5488 100755 --- a/web/build.sh +++ b/web/build.sh @@ -3,7 +3,6 @@ # Compiles the Keyman Engine for Web and its various end-products # -# set -x set -eu ## START STANDARD BUILD SCRIPT INCLUDE @@ -23,15 +22,16 @@ cd "$THIS_SCRIPT_PATH" # Ensures that we rely first upon the local npm-based install of Typescript. # (Facilitates automated setup for build agents.) +# TODO: this should be removeable given set_keyman_standard_build_path does this in build-utils.sh (and relative paths are dodgy in $PATH!) PATH="../node_modules/.bin:$PATH" PREDICTIVE_TEXT_SOURCE="../common/predictive-text/unit_tests/in_browser/resources/models/simple-trie.js" PREDICTIVE_TEXT_OUTPUT="src/test/manual/web/prediction-ui/simple-en-trie.js" builder_describe "Builds Keyman Engine for Web (KMW)." \ - "@../common/web/keyman-version build" \ - "@../common/web/input-processor build" \ - "@src/tools/building/sourcemap-root build" \ + "@/common/web/keyman-version build" \ + "@/common/web/input-processor build" \ + "@/web/src/tools/building/sourcemap-root build" \ "clean" \ "configure" \ "build" \ @@ -49,19 +49,19 @@ builder_describe "Builds Keyman Engine for Web (KMW)." \ # "upload-symbols Uploads build product to Sentry for error report symbolification. Only defined for $(builder_term build:embed) and $(builder_term build:web)" \ builder_describe_outputs \ - configure ../node_modules \ - configure:embed ../node_modules \ - configure:engine ../node_modules \ - configure:web ../node_modules \ - configure:ui ../node_modules \ - configure:samples ../node_modules \ - configure:tools ../node_modules \ - build:embed build/app/embed/release/keyman.js \ - build:engine build/engine/main/obj/keymanweb.js \ - build:web build/app/web/release/keymanweb.js \ - build:ui build/app/ui/release/kmwuibutton.js \ - build:samples $PREDICTIVE_TEXT_OUTPUT \ - build:tools build/tools/building/sourcemap-root/index.js + configure /node_modules \ + configure:embed /node_modules \ + configure:engine /node_modules \ + configure:web /node_modules \ + configure:ui /node_modules \ + configure:samples /node_modules \ + configure:tools /node_modules \ + build:embed /web/build/app/embed/release/keyman.js \ + build:engine /web/build/engine/main/obj/keymanweb.js \ + build:web /web/build/app/web/release/keymanweb.js \ + build:ui /web/build/app/ui/release/kmwuibutton.js \ + build:samples /web/$PREDICTIVE_TEXT_OUTPUT \ + build:tools /web/build/tools/building/sourcemap-root/index.js builder_parse "$@" @@ -293,7 +293,7 @@ copy_outputs ( ) { # ``` compile ( ) { local COMPILE_TARGET=$1 - npm run tsc -- -b src/$COMPILE_TARGET -v || builder_die "Build command tsc -- -b src/$COMPILE_TARGET -v failed with exit code $?" + tsc -b src/$COMPILE_TARGET -v || builder_die "Build command tsc -b src/$COMPILE_TARGET -v failed with exit code $?" echo $COMPILE_TARGET TypeScript compiled under build/$COMPILE_TARGET/obj } diff --git a/web/src/engine/build.sh b/web/src/engine/build.sh index 0a3b571fae..36ba484bf3 100755 --- a/web/src/engine/build.sh +++ b/web/src/engine/build.sh @@ -72,18 +72,22 @@ output_path ( ) { fi } +# TODO: eliminate $SOURCE variable SOURCE="src" # Ensures that we rely first upon the local npm-based install of Typescript. # (Facilitates automated setup for build agents.) +# TODO: this should be removeable given set_keyman_standard_build_path does this in build-utils.sh (and relative paths are dodgy in $PATH!) PATH="../../../node_modules/.bin:$PATH" +# TODO: call tsc directly -- but currently it runs from wrong folder if called directly +# TODO: eliminate $compiler and $compilecmd variables compiler="npm run tsc --" compilecmd="$compiler" builder_describe "Builds engine modules for Keyman Engine for Web (KMW)." \ - "@../../../common/web/keyman-version build:main" \ - "@../../../common/web/input-processor build:main" \ + "@/common/web/keyman-version build:main" \ + "@/common/web/input-processor build:main" \ "clean" \ "configure" \ "build" \ @@ -95,13 +99,14 @@ builder_describe "Builds engine modules for Keyman Engine for Web (KMW)." \ # "upload-symbols Uploads build product to Sentry for error report symbolification. Only defined for $DOC_BUILD_EMBED_WEB" \ builder_describe_outputs \ - configure ../../../node_modules \ - configure:device-detect ../../../node_modules \ - configure:element-wrappers ../../../node_modules \ - configure:main ../../../node_modules \ + configure /node_modules \ + configure:device-detect /node_modules \ + configure:element-wrappers /node_modules \ + configure:main /node_modules \ build:device-detect $(output_path $DEVICEDETECT $OUTPUT_DIR)/index.js \ build:element-wrappers $(output_path $ELEMENTWRAPPERS $OUTPUT_DIR)/index.js \ build:main $(output_path $MAIN $OUTPUT_DIR)/keymanweb.js +# TODO: eliminate outpath_path function and related variables and just use the plain filenames builder_parse "$@" diff --git a/web/src/tools/build.sh b/web/src/tools/build.sh index a9dd7f45d4..877aecf55d 100755 --- a/web/src/tools/build.sh +++ b/web/src/tools/build.sh @@ -18,8 +18,8 @@ cd "$THIS_SCRIPT_PATH" ################################ Main script ################################ builder_describe "Builds the Keyman Engine for Web's development & unit-testing tools" \ - "@../../../common/web/keyman-version" \ - "@../../../common/web/keyboard-processor" \ + "@/common/web/keyman-version" \ + "@/common/web/keyboard-processor" \ "clean" \ "build" \ ":bulk_rendering Builds the bulk-rendering tool used to validate changes to OSK display code" \ diff --git a/web/src/tools/building/sourcemap-root/build.sh b/web/src/tools/building/sourcemap-root/build.sh index abed644429..ec2f5d8b09 100755 --- a/web/src/tools/building/sourcemap-root/build.sh +++ b/web/src/tools/building/sourcemap-root/build.sh @@ -18,14 +18,14 @@ cd "$THIS_SCRIPT_PATH" ################################ Main script ################################ builder_describe "Builds the sourcemap-sanitizing script used for Keyman Engine for Web builds" \ - "@../../../../../common/tools/sourcemap-path-remapper" \ + "@/common/tools/sourcemap-path-remapper" \ "clean" \ "configure" \ "build" \ builder_describe_outputs \ configure /node_modules \ - build ../../../../build/tools/sourcemap-root/index.js + build /web/build/tools/sourcemap-root/index.js builder_parse "$@" @@ -42,7 +42,7 @@ if builder_start_action clean; then fi if builder_start_action build; then - npm run tsc -- -b "$THIS_SCRIPT_PATH/tsconfig.json" + tsc -b "$THIS_SCRIPT_PATH/tsconfig.json" # Necessary until the Web project is converted over to ES modules. # Node defaults to CommonJS format otherwise. diff --git a/web/src/tools/testing/bulk_rendering/build.sh b/web/src/tools/testing/bulk_rendering/build.sh index 40e4e6086f..951eaeb56c 100755 --- a/web/src/tools/testing/bulk_rendering/build.sh +++ b/web/src/tools/testing/bulk_rendering/build.sh @@ -27,7 +27,7 @@ builder_describe \ builder_describe_outputs \ configure /node_modules \ - build ../../../../build/tools/testing/bulk_rendering/bulk_render.js + build /web/build/tools/testing/bulk_rendering/bulk_render.js builder_parse "$@" diff --git a/web/src/tools/testing/recorder/build.sh b/web/src/tools/testing/recorder/build.sh index 8124665856..dfe0d600d7 100755 --- a/web/src/tools/testing/recorder/build.sh +++ b/web/src/tools/testing/recorder/build.sh @@ -18,16 +18,16 @@ cd "$THIS_SCRIPT_PATH" ################################ Main script ################################ builder_describe "Builds the Keyman Engine for Web's test-sequence recording tool" \ - "@../../../../../common/web/keyman-version" \ - "@../../../../../common/web/keyboard-processor" \ - "@../../../../../common/web/recorder" \ + "@/common/web/keyman-version" \ + "@/common/web/keyboard-processor" \ + "@/common/web/recorder" \ "clean" \ "configure" \ "build" builder_describe_outputs \ configure /node_modules \ - build ../../../../build/tools/testing/recorder/index.js + build /web/build/tools/testing/recorder/index.js builder_parse "$@" @@ -48,7 +48,7 @@ fi ### BUILD ACTIONS if builder_start_action build; then - npm run tsc -- -b $THIS_SCRIPT_PATH/tsconfig.json + tsc -b $THIS_SCRIPT_PATH/tsconfig.json builder_finish_action success build fi \ No newline at end of file From 578b79e739824dc819325b8ef095d3e555177c73 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 13 Mar 2023 08:08:10 +0700 Subject: [PATCH 04/12] chore(common): fix typo --- common/web/lm-worker/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/web/lm-worker/build.sh b/common/web/lm-worker/build.sh index cc6aaf5162..59472b0f93 100755 --- a/common/web/lm-worker/build.sh +++ b/common/web/lm-worker/build.sh @@ -113,7 +113,7 @@ if builder_start_action build; then fi # Build worker with tsc first - tsb -b $builder_verbose || builder_die "Could not build worker." + tsc -b $builder_verbose || builder_die "Could not build worker." # Wrap the worker code and create embedded index.js. Must be run after the # worker is built From de0580f4363f2249a17135e96969e028d893da3d Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 13 Mar 2023 08:16:45 +0700 Subject: [PATCH 05/12] chore(common): fix another typo --- resources/builder.inc.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/builder.inc.sh b/resources/builder.inc.sh index 2d484b02f3..7e4c0842ee 100755 --- a/resources/builder.inc.sh +++ b/resources/builder.inc.sh @@ -889,11 +889,11 @@ function builder_describe_outputs() { _builder_record_function_call builder_describe_outputs while [[ $# -gt 0 ]]; do - local key="$1" path="$2" action target + local key="$1" path="$2" action= target= path="`_builder_expand_relative_path "$path"`" if [[ $key =~ : ]]; then - IFS=":" read -r -a action target <<< "$key" + IFS=":" read -r action target <<< "$key" target=":$target" else # Add dependency expected output file for all targets, as well as a From f41922d7490d541efe8c2ddbe692e646155c7e70 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 13 Mar 2023 08:30:38 +0700 Subject: [PATCH 06/12] docs: build-utils standard environment --- resources/build/build-utils.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/resources/build/build-utils.md b/resources/build/build-utils.md index fb77e638ad..41f570994e 100644 --- a/resources/build/build-utils.md +++ b/resources/build/build-utils.md @@ -113,6 +113,15 @@ following line here: cd "$THIS_SCRIPT_PATH" ``` +## Standard environment + +`build-utils.sh` will prepend `$KEYMAN_ROOT/node_modules/.bin` to the `PATH` +variable to ensure that we run the correct versions of node-based commands, so +there is no need to hard-code path references or add script wrappers to +package.json (`npm run