# shellcheck shell=bash # shellcheck disable=SC2310 # Note: these two lines can be uncommented for debugging and profiling build # scripts: # # set -x # PS4='+ $EPOCHREALTIME $0 $LINENO ' # # # This script contains utilities for builder_script calls # # * builder_ functions and variables are defined here. # * REPO_ROOT defines the top level of this repository # * THIS_SCRIPT_PATH defines the full path of the running script # * THIS_SCRIPT_NAME defines the basename of the running script # * THIS_SCRIPT_IDENTIFIER defines the repo-relative path of the running script # * _builder_ functions and variables are internal use only for builder.inc.sh, and # subject to change at any time. Do not use them in other scripts. # * Note: the running script is the top-level script that includes either # builder.inc.sh directly, or, just in the Keyman repo, via builder-basic.inc.sh. # # Exit on command failure and when using unset variables: set -eu # # Prevents 'clear' on exit of mingw64 bash shell # SHLVL=0 # A regex of reserved builder parameters, must be kept in sync with # the matching of builder parameters in _builder_parse_expanded_parameters _builder_reserved_parameters='^(--help|-h|--color|--no-color|--verbose|-v|--timings|--no-timings|--debug|--release|--deps|--no-deps|--force-deps|--builder-dep-parent|--builder-child|--builder-report-dependencies|--builder-completion-describe|--offline|--builder-ignore-unknown-options)$' # _builder_init is called internally at the bottom of this file after we have # all function declarations in place. function _builder_init() { _builder_get_operating_system _builder_findRepoRoot _builder_setBuildScriptIdentifiers if [[ -n "$TERM" ]] && [[ "$TERM" != "dumb" ]] && [[ "$TERM" != "unknown" ]] && [ -t 1 ]; then builder_use_color true else builder_use_color false fi # Set shared Meson package cache (works with Meson >= 1.3) if [[ -z ${MESON_PACKAGE_CACHE_DIR:-} ]]; then export MESON_PACKAGE_CACHE_DIR="${XDG_CACHE_HOME:-${HOME}/.cache}/keyman/builder" mkdir -p "${MESON_PACKAGE_CACHE_DIR}" fi } function _builder_findRepoRoot() { REPO_ROOT="$(readlink -f "${BASH_SOURCE[0]%/*/*}")" readonly REPO_ROOT } # Used to build script-related build variables useful for referencing the calling script # and for prefixing `builder_finish_action` outputs in order to more clearly identify the calling # script. # # Assumes that `THIS_SCRIPT` has been set, typically like this: # # ```bash # ## START STANDARD BUILD SCRIPT INCLUDE # # adjust relative paths as necessary # THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" # . "${THIS_SCRIPT%/*}/resources/builder.inc.sh" # ## END STANDARD BUILD SCRIPT INCLUDE # ``` # function _builder_setBuildScriptIdentifiers() { if [ ! -z ${THIS_SCRIPT+x} ]; then THIS_SCRIPT_PATH="${THIS_SCRIPT%/*}" readonly THIS_SCRIPT_PATH THIS_SCRIPT_NAME="${THIS_SCRIPT##*/}" readonly THIS_SCRIPT_NAME # Leaves only the part of the path based upon REPO_ROOT. THIS_SCRIPT_IDENTIFIER=${THIS_SCRIPT_PATH#"$REPO_ROOT/"} readonly THIS_SCRIPT_IDENTIFIER else builder_die "THIS_SCRIPT not defined; builder.inc.sh has not been sourced with standard script include." fi } ################################################################################ # Standard build script functions for managing command line, actions and targets ################################################################################ # The following allows coloring of warning and error lines, but only works if there's a # terminal attached, so not on the build machine. # Overrides default colorization of logging; can be used in command-line with # --color or --no-color, or overridden as necessary on a per-script basis. # # Parameters # 1: use_color true or false builder_use_color() { if $1; then # Using esc codes instead of tput for performance COLOR_RED='\x1b[31m' # $(tput setaf 1) COLOR_GREEN='\x1b[32m' # $(tput setaf 2) COLOR_YELLOW='\x1b[33m' # $(tput setaf 3) COLOR_BLUE='\x1b[34m' # $(tput setaf 4) COLOR_PURPLE='\x1b[35m' # $(tput setaf 5) COLOR_TEAL='\x1b[36m' # $(tput setaf 6) COLOR_WHITE='\x1b[38;5;252m' # $(tput setaf 252) COLOR_BRIGHT_WHITE='\x1b[38;5;255m' # $(tput setaf 255) COLOR_GREY='\x1b[90m' # $(tput setaf 8) COLOR_RESET='\x1b(B\x1b[m' # $(tput sgr0) # e.g. VSCode https://code.visualstudio.com/updates/v1_69#_setmark-sequence-support BUILDER_BOLD='\x1b[1m' # $(tput bold) HEADING_SETMARK='\x1b]1337;SetMark\x07' # Used by `builder_display_usage` when marking special terms (actions, targets, options) # in the plain-text description area. BUILDER_TERM_START="$COLOR_BLUE" BUILDER_TERM_END="$COLOR_RESET" else COLOR_RED= COLOR_GREEN= COLOR_YELLOW= COLOR_BLUE= COLOR_PURPLE= COLOR_TEAL= COLOR_WHITE= COLOR_BRIGHT_WHITE= COLOR_GREY= COLOR_RESET= BUILDER_BOLD= HEADING_SETMARK= BUILDER_TERM_START="<" BUILDER_TERM_END=">" fi } # # Wraps the input string in `builder_display_usage` with $BUILDER_TERM_START and # $BUILDER_TERM_END # function builder_term() { echo -e "${BUILDER_TERM_START}$*${BUILDER_TERM_END}" } function builder_die() { _builder_error_echo if [[ $# -eq 0 ]]; then builder_echo error "Unspecified error, aborting script" else builder_echo error "$*" fi _builder_error_echo exit 1 } # Emit message to stderr instead of stdout function _builder_error_echo() { # we only need to support -e flag if [[ $# -gt 0 ]] && [[ $1 == -e ]]; then shift 2>&1 echo -e "$*" else 2>&1 echo "$*" fi } function builder_warn() { builder_echo warning "$*" } function builder_heading() { builder_echo heading "$*" } #################################################################################### # # builder_ functions for standard build script parameter and process management # #################################################################################### builder_echo() { local color=white message= mark= block= action= test= local echo_target=echo if [[ $# -gt 1 ]]; then if [[ $1 =~ ^(white|grey|green|success|blue|heading|yellow|warning|red|error|purple|brightwhite|teal|debug|setmark)$ ]]; then color="$1" shift elif [[ $1 == "start" ]] || [[ $1 == "startTest" ]]; then # builder_echo start block message test="$1" block="$2" shift 2 action="start" color="heading" elif [[ $1 == "end" ]] || [[ $1 == "endTest" ]]; then # builder_echo end block status message test="$1" block="$2" color="$3" shift 3 action="end" fi fi message="$*" if [[ "${action}" == "start" ]] && builder_is_running_on_teamcity; then if [[ "${test}" == "startTest" ]]; then $echo_target -e "##teamcity[testSuiteStarted name='|[${THIS_SCRIPT_IDENTIFIER}|] ${block}']" else $echo_target -e "##teamcity[blockOpened name='|[${THIS_SCRIPT_IDENTIFIER}|] ${block}']" fi fi if [[ ! -z ${COLOR_RED+x} ]]; then case $color in white) color="$COLOR_WHITE" ;; grey) color="$COLOR_GREY" ;; green|success) color="$COLOR_GREEN" ;; blue|heading) color="$COLOR_BLUE" ;; yellow|warning) color="$COLOR_YELLOW" ;; red|error) color="$COLOR_RED"; echo_target=_builder_error_echo ;; purple) color="$COLOR_PURPLE" ;; brightwhite) color="$COLOR_BRIGHTWHITE" ;; teal|debug) color="$COLOR_TEAL" ;; setmark) mark="$HEADING_SETMARK" color="$COLOR_PURPLE" ;; esac if builder_is_dep_build; then $echo_target -e "$mark$COLOR_GREY[$THIS_SCRIPT_IDENTIFIER]$COLOR_RESET $color$message$COLOR_RESET" else $echo_target -e "$mark$BUILDER_BOLD$COLOR_BRIGHT_WHITE[$THIS_SCRIPT_IDENTIFIER]$COLOR_RESET $color$message$COLOR_RESET" fi else # Cope with the case of pre-init message and just emit plain text $echo_target -e "$message" fi if [[ "${action}" == "end" ]] && builder_is_running_on_teamcity; then if [[ "${test}" == "endTest" ]]; then $echo_target -e "##teamcity[testSuiteFinished name='|[${THIS_SCRIPT_IDENTIFIER}|] ${block}']" else $echo_target -e "##teamcity[blockClosed name='|[${THIS_SCRIPT_IDENTIFIER}|] ${block}']" fi fi } builder_echo_debug() { builder_echo debug "[DEBUG] $*" } # # builder_ names are reserved. # _builder_ names are internal use and subject to change # # # builder_extra_params: string containing all parameters after '--' # builder_extra_params=() # returns 0 if first parameter is in the array passed as second parameter # # Usage: # if _builder_item_in_array "item" "${array[@]}"; then ...; fi # Parameters: # 1: item item to search for in array # 2: array bash array, e.g. array=(one two three) _builder_item_in_array() { local e match="$1" shift [[ -z "$match" ]] && return 1 for e; do [[ "$e" == $match ]] && return 0; done return 1 } # # Returns `0` if first parameter is in the array passed as second parameter, # where the array may contain globs. # # ### Parameters # # * 1: `item` item to search for in array # * 2: `array` bash array, e.g. `array=(one two three)` # # ### Example # # ```bash # array=(foo bar it*) # if _builder_item_in_glob_array "item" "${array[@]}"; then ...; fi # ``` # _builder_item_in_glob_array() { local e match="$1" shift [[ -z "$match" ]] && return 1 for e; do [[ "$match" == $e ]] && return 0; done return 1 } # # Expands a shorthand item into a full match from an array of possibilities; # reports an error if there are ambiguous options. Note that this function # returns the number of matches, so 0 = no match, 1 = a precise match. # # ### Parameters # # * 1: `item` item to search for in array, e.g. "t" # * 2: `array` bash array, e.g. `array=(one two three)` # # ### Description # # Does a substring search by regex for # # ### Example # # ```bash # actions=(clean configure build test) # # action=`_builder_expand_shorthand $1 "${actions[@]}"` && # builder_die "Unrecognized parameter $1" || # case $? in # 1) echo "Parameter $1 matches {$action}" # ;; # *) builder_die "Parameter $1 has $? matches, could mean any of {$action}" # esac # ``` # _builder_expand_shorthand() { local item=$1 shift local count=0 local result= local string= for e; do if [[ $e == $item ]]; then # Exact match trumps substring matches echo $item return 1 fi if [[ $e == "$item"* ]]; then count=$((count+1)) if [[ $count == 2 ]]; then string="$result, $e" result=$item elif [[ $count -gt 2 ]]; then string="$string, $e" else result=$e fi fi done if [[ $count -lt 2 ]]; then echo $result else echo $string fi return $count } _builder_item_is_target() { local item="$1" [[ $item =~ ^: ]] && return 1 return 0 } function _builder_warn_if_incomplete() { if [ -n "${_builder_current_action}" ]; then builder_echo warning "WARNING: $_builder_current_action never reported success or failure" # exit 1 # If we wanted this scenario to result in a forced build-script fail. fi # Since we've already warned about this once, we'll clear the variable to prevent repetitions. _builder_current_action= } # Used by a `trap` statement later to facilitate auto-reporting failures on error detection # without obscuring failure exit/error codes. _builder_failure_trap() { local trappedExitCode=$? local action target _builder_cleanup_deps _builder_timing_cleanup_and_report $trappedExitCode # Since 'exit' is also trapped, we can also handle end-of-script incomplete actions. if [[ $trappedExitCode == 0 ]]; then # While there weren't errors, were there any actions that never reported success or failure? _builder_warn_if_incomplete return fi # If we've reached this point, we're here because an error occurred. # Iterate across currently-active actions and report their failures. if [ -n "${_builder_current_action}" ]; then action="${_builder_current_action}" if [[ $action =~ : ]]; then IFS=: read -r action target <<< "$action" target=:$target else target=:project fi builder_finish_action failure $action$target fi # Make 100% sure that the exit code chains fully. # Without this, nested scripts have failed to chain errors from npm calls past the script # that directly executed the failed npm command. exit $trappedExitCode } # # Removes temporary `_builder_deps_built` file when top-level build script # finishes. # _builder_cleanup_deps() { if ! builder_is_dep_build && ! builder_is_child_build && [[ ! -z ${_builder_deps_built+x} ]]; then if $_builder_debug_internal; then builder_echo_debug "Dependencies that were built:" cat "$_builder_deps_built" fi rm -f "$_builder_deps_built" _builder_deps_built= fi } #------------------------------------------------------------------------------------------ # Child scripts #------------------------------------------------------------------------------------------ # # Starts a child script build, passing current build dependency, inheritable # options, and timing status through as parameters. This should be used rather # than calling the build script directly, to avoid multiple builds of # dependencies. # # Do not use builder standard options such as `--deps` or `--debug`. # # ### Parameters # # * 1: `script` path to script, relative to root of repo, with leading slash # * 2+: `parameters` action(s), target(s), parameters for the child script to run # # ### Example # # ```bash # builder_launch /core/build.sh configure,build:wasm --no-tests # ``` # builder_launch() { local script="$1" local param shift if [[ ! "$script" =~ ^/(.+) ]]; then builder_die "Error: builder_launch: script path must start with /, and is relative to repo root" fi for param in "$@"; do if [[ "${param}" =~ ${_builder_reserved_parameters} ]]; then builder_die "Error: builder_launch: reserved parameter '${param}' used" fi done builder_echo grey "## script '$script $*' launched..." _builder_execute_child_script "${KEYMAN_ROOT}${script}" "$@" builder_echo grey "## script '$script $*' completed" } _builder_execute_child() { local action="$1" local target="$2" local script="$THIS_SCRIPT_PATH/${_builder_target_paths[$target]}/build.sh" builder_echo grey "## child $action$target starting..." _builder_execute_child_script "$script" "$action" builder_echo grey "## child $action$target completed successfully" } # ### Parameters # # * 1: `script` path to script # * 2+: `parameters` action(s), target(s), parameters for the child script to run _builder_execute_child_script() { local script="$1" # subsequent parameters passed as $@/$* below shift # Build array of specified inheritable options local child_options=() local opt for opt in "${_builder_options_inheritable[@]}"; do if builder_has_option "${opt}"; then child_options+=("${opt}") fi done # If the current build is a dependency build, pass the dependency state on to # the child build, so that we don't unnecessarily rebuild children (#11394) local dep_flag= dep_module= if builder_is_dep_build; then dep_flag=--builder-dep-parent dep_module="$builder_dep_parent" fi "$script" \ --builder-child \ $_builder_build_deps \ $dep_flag "$dep_module" \ "$@" \ "${child_options[@]}" \ $builder_verbose \ $builder_debug \ $_builder_offline \ || ( result=$? builder_echo error "## child script '$script $*' failed with exit code $result" exit $result ) || exit $? # Required due to above subshell masking exit } _builder_run_child_action() { local action="$1" target if [[ $action =~ : ]]; then IFS=: read -r action target <<< "$action" target=:$target else target=':*' fi if builder_has_action $action$target; then if [[ $target == ':*' ]]; then # run all children in order specified in builder_describe for target in "${_builder_targets[@]}"; do # We have to re-test the action because the user may not # have specified all targets in their invocation if builder_has_action $action$target; then if [[ ! -z ${_builder_target_paths[$target]+x} ]] && [[ -f "$THIS_SCRIPT_PATH/${_builder_target_paths[$target]}/build.sh" ]]; then _builder_execute_child $action $target fi fi done else # If specified explicitly, we assume existence of a child build script. _builder_execute_child $action $target fi fi } # # Executes the specified actions on all child targets, or on the specified # targets. A child target is any target which has a sub-folder of the same name # as the target. However, the actions will only be run if they have been # specified by the user on the command-line. # # ### Usage # # ```bash # builder_run_child_actions action1 [...] # ``` # # ### Parameters # # 1...: action[:target] name of action:target to run # # ### Example # # ```bash # builder_run_child_actions configure build test install # ``` # builder_run_child_actions() { while [[ $# -gt 0 ]]; do local action="$1" _builder_run_child_action "$action" shift done } #------------------------------------------------------------------------------------------ # Various API endpoints #------------------------------------------------------------------------------------------ # # Builds the standardized `action:target` string for the specified action-target # pairing and also returns 0 if the user has asked to perform it on the command # line. Otherwise, returns 0 and sets an empty string in place of the matched # pair. # # The string will be set as `_builder_matched_action`, which is for # builder.inc.sh internal use, used by `builder_start_action`. # # ### Usage # # ```bash # if build_has_action action[:target]; then ...; fi # ```` # # ### Parameters # # 1: action[:target] name of action:target # # ### Example # # ```bash # if builder_has_action build:app; then ... # ``` # builder_has_action() { local action="$1" target if [[ $action =~ : ]]; then IFS=: read -r action target <<< "$action" target=:$target else target=':*' fi if _builder_item_in_array "$action$target" "${_builder_chosen_action_targets[@]}"; then # To avoid WET re-processing of the $action$target string set _builder_matched_action="$action$target" if [[ $target == ':*' ]]; then _builder_matched_action_name="$action" else _builder_matched_action_name="$action$target" fi return 0 else _builder_matched_action= return 1 fi } # # Wraps builder_start_action and builder_finish action for single-command # actions. Can be used together with a local function for multi-command actions. # Do be aware that this pseudo-closure style cannot be mixed with operators such # as `<`, `>`, `&&`, `;`, `()` and so on. # # ### Usage # # ```bash # builder_run_action action[:target] command [command-params...] # ``` # # ### Parameters # # * 1: `action[:target]` name of action, and optionally also target, if target # excluded starts for all defined targets # * 2: command command to run if action is started # * 3...: command-params parameters for command # # ### Example # # ```bash # function do_build() { # mkdir -p build/cjs-src # npm run build # } # # builder_run_action clean rm -rf ./build/ ./tsconfig.tsbuildinfo # builder_run_action configure node_select_version_and_npm_ci # builder_run_action build do_build # ``` # function builder_run_action() { local action=$1 shift if builder_start_action $action; then "$@" builder_finish_action success $action fi return 0 } # # Returns `0` if the user has asked to perform action on target on the command # line, and then starts the action. Should be paired with # `builder_finish_action`. # # ### Usage # # ```bash # if builder_start_action action[:target]; then ...; fi # ``` # # ### Parameters # # * 1: `action[:target]` name of action, and optionally also target, if # target excluded starts for all defined targets # # ### Example # # ```bash # if builder_start_action build:app; then ... # ``` # builder_start_action() { if builder_has_action $1; then # In a dependency quick build (the default), determine whether we actually # need to run this step. Uses data passed to builder_describe_outputs to # verify whether a target output is present. if builder_is_dep_build && ! builder_is_full_dep_build && _builder_dep_output_exists "${_builder_matched_action}"; then builder_echo "skipping ${_builder_matched_action_name}, up-to-date" return 1 fi builder_echo start "${_builder_matched_action_name}" "## ${_builder_matched_action_name} starting..." if [[ -n "${_builder_current_action}" ]]; then _builder_warn_if_incomplete fi _builder_current_action="${_builder_matched_action}" # Note: we include the action's dependencies in timing _builder_timing_start "${_builder_matched_action}" # Build dependencies as required _builder_do_build_deps "${_builder_matched_action}" return 0 else return 1 fi } ################################################################################################### # Timing ################################################################################################### _builder_timing_setup() { if $_builder_timings; then export _builder_timing_report_file=`mktemp` _builder_timing_start "build.sh:top" fi } _builder_timing_start() { local context="$1" if [[ ! -z "${_builder_timing_report_file:+x}" ]]; then echo "START $THIS_SCRIPT_IDENTIFIER:$context $(date +%s.%N)" >> "$_builder_timing_report_file" fi } _builder_timing_stop() { local context="$1" if [[ ! -z "${_builder_timing_report_file:+x}" ]]; then echo "STOP $THIS_SCRIPT_IDENTIFIER:$context $(date +%s.%N)" >> "$_builder_timing_report_file" fi } # # Removes temporary `_builder_timing_report_file` file when top-level build # script finishes and reports on timings if they were requested. # _builder_timing_cleanup_and_report() { local build_exit_code="$1" if builder_is_dep_build || builder_is_child_build || [[ -z ${_builder_timing_report_file:+x} ]]; then return 0 fi if [[ "$build_exit_code" != 0 ]]; then # Hide the timings report if build failed, but still cleanup _builder_timings=false fi if $_builder_timings; then _builder_timing_stop "build.sh:top" if ! command -v node > /dev/null; then builder_warn "WARNING: node.js not found, skipping timings report" else builder_heading "## Build timings" # for simplicity of distribution, we embed the node.js script here # shellcheck disable=SC2016 node -e ' const fs = require("node:fs"); const process = require("node:process"); let COLOR_RED="", COLOR_GREEN="", COLOR_YELLOW="", COLOR_BLUE="", COLOR_PURPLE="", COLOR_TEAL="", COLOR_WHITE="", COLOR_BRIGHT_WHITE="", COLOR_GREY="", COLOR_RESET="", BUILDER_BOLD=""; if(process.stdout.isTTY) { COLOR_RED="\x1b[31m"; COLOR_GREEN="\x1b[32m"; COLOR_YELLOW="\x1b[33m"; COLOR_BLUE="\x1b[34m"; COLOR_PURPLE="\x1b[35m"; COLOR_TEAL="\x1b[36m"; COLOR_WHITE="\x1b[38;5;252m"; COLOR_BRIGHT_WHITE="\x1b[38;5;255m"; COLOR_GREY="\x1b[90m"; COLOR_RESET="\x1b(B\x1b[m"; BUILDER_BOLD="\x1b[1m"; } const log = fs.readFileSync(process.argv[process.argv.length-1], "utf8").split("\n"); let top = null; const modules = {}; const tree = []; for(const line of log) { if(line.trim() == "") continue; const [event, area, time] = line.split(" "); const [module, action, target, sub] = area.split(":"); if(event == "START") { const mod = {area, module, action, target: target ? ":"+target : "", sub: sub ?? "", start:parseFloat(time), stop:null, modules:[]}; if(!top) top = mod; modules[area] = mod; if(tree.length) { tree[tree.length-1].modules.push(mod); } tree.push(mod); } else if(event == "STOP") { if(tree.length) { const mod = tree.pop(); if(mod.area != area) { console.error(`Expected ${area} to equal ${mod.area}`); } } const mod = modules[area]; if(!mod) { console.error(`START not found for ${area}`); } else { mod.stop = parseFloat(time); } } else { console.error(`Unrecognized line: "${line}"`); } } if(tree.length) { console.error(`Expected tree to be empty`); console.dir(tree); } if(!Object.keys(modules).length || !top) { console.error(`Expected report to be non-empty`); process.exit(0); } // Write out the tree with Pretty Printing printModule("", top); function printModule(indent, mod) { const duration = Math.round((mod.stop - mod.start) * 10) / 10; console.log(`${indent}${BUILDER_BOLD}[${mod.module}]${COLOR_RESET} ${COLOR_GREEN}${mod.action}${mod.target}${COLOR_RESET} ${mod.sub} ${duration}`); for(const submod of mod.modules) { printModule(indent+" ", submod); } } ' "${_builder_timing_report_file}" fi fi rm -f "$_builder_timing_report_file" _builder_timing_report_file= } ################################################################################################### # # Returns 0 if the user has --option on the command line # # Usage: # if build_has_option option; then ...; fi # Parameters: # 1: option name of option, i.e. --option # Example: # if build_has_option --debug; then # builder_has_option() { local option="$1" if _builder_item_in_array "$option" "${_builder_chosen_options[@]}"; then return 0 fi return 1 } # # Trims leading and following whitespace from the input parameters # # ### Usage # # ```bash # my_string="$(builder_trim "$my_string")" # ``` # # ### Parameters # # * `my_string` An input string # builder_trim() { local var="$*" # remove leading whitespace characters var="${var#"${var%%[![:space:]]*}"}" # remove trailing whitespace characters var="${var%"${var##*[![:space:]]}"}" printf '%s' "$var" } # # Expands an in-repo-relative path to a repo-relative path. A path starting with # `/` is expected to be relative to repo root, not filesystem root. Otherwise, # it's relative to current script path, not current working directory. The # returned path will not have a prefix `/`, and will be relative to # `$REPO_ROOT`. Assumes realpath is installed (brew coreutils on macOS). # _builder_expand_relative_path() { local path="$1" if [[ "$path" =~ ^/ ]]; then echo "${path:1}" else realpath --canonicalize-missing --relative-to="$REPO_ROOT" "$THIS_SCRIPT_PATH/$path" fi } # # Expands an `[action][:target]` string, replacing missing values with `*`, # for example: # # * `build` --> `build:*` # * `build:app` --> `build:app` # * `:app` --> `*:app` # # Supports multiple action:targets in the string # _builder_expand_action_target() { local input="$1" target= action= if [[ "$input" =~ : ]]; then IFS=":" read -r action target <<< "$input" else action="$input" fi if [[ -z "$action" ]]; then action='*' fi if [[ -z "$target" ]]; then target='*' fi echo "$action:$target" } _builder_expand_action_targets() { local input=($1) e output=() for e in "${input[@]}"; do e="$(_builder_expand_action_target "$e")" output+=("$e") done if [[ ${#output[@]} == 0 ]]; then echo "*:*" else echo "${output[@]}" fi } _builder_child_base= # # Describes the path from the build script's working directory to the common subfolder # containing child scripts / projects without defined custom paths. # # This function must be called to set the child base path before builder_describe is # called in order to work correctly. Furthermore, note that this setting will be # ignored by targets with custom paths. # # ### Usage # # ```bash # builder_set_child_base path # ``` # # ### Parameters # # * `path` The relative path from the directory containing the calling script to # the base folder to use for child-project detection and resolution # builder_set_child_base() { _builder_child_base="$1/" } # # Describes a build script, defines available parameters and their meanings. Use # together with `builder_parse` to process input parameters. # # ### Usage # # ```bash # builder_describe description param_desc... # ``` # # ### Parameters # # * `description` A short description of what the script does. # * `param_desc` Space separated name and description of parameter, e.g. # `"build Builds the target"` # This parameter may be repeated to describe all parameters. # # There are four types of parameters that may be specified: # # * **Option:** `"--option[,-o][+][=var] [One line description]"` # # All options must have a longhand form with two prefix hyphens, # e.g. `--option`. The `,-o` shorthand form is optional. When testing if # the option is set with `builder_has_option`, always use the longhand # form. # # If `=var` is specified, then the next parameter will be a variable stored in # `$var` for that option. e.g. `--option=opt` means `$opt` will have the value # `"foo"` when the script is called for `--option foo`. # # If `+` is specified, then the option will be passed to child scripts. All # child scripts _must_ accept this option, or they will fail. It is acceptable # for the child script to declare the option but ignore it. However, the option # will _not_ be passed to dependencies. # # * **Action**: `"action [One line description]"` # # Actions must be a single word, lower case. To specify an action as the # default, append a `+` to the action name, e.g. `"test+ Test the project"`. # If there is no default specified, then it will be `build`. # # * **Target:** `":target[=path] [One line description]"` # # A target always starts with colon, e.g. `:project`. If a folder exists with # the same name as a target, then that automatically denotes the target as a # "child project". This can simplify parent-child style scripts, using the # [`builder_run_child_actions`] function. # # A child project with an alternate folder can also be specified by appending # `=path` to the target definition, for example `:app=src/app`. Where # possible, avoid differences in names of child projects and folders. # # * **Dependency:** `"@/path/to/dependency[:target] [action][:target] ..."`` # # A dependency always starts with `@`. The path to the dependency will be # relative to the build script folder, or to the root of the repository, if # the path starts with `/`, not to the root of the file system. It is an error # to specify a dependency outside the repo root. # # Relative paths will be expanded to full paths, again, relative to the root # of the repository. # # A dependency definition can include a target for that dependency, for # example, `"@/core:arch"`. This would build only the ':arch' target for the # core module. # # Dependencies may be limited to specific `action:target` pairs on the current # script. If not specified, dependencies will be built for all actions on all # targets. Either `action` or `:target` may be omitted, and multiple actions # and targets may be specified, space separated. # builder_describe() { _builder_record_function_call builder_describe _builder_description="$1" _builder_actions=() _builder_targets=() _builder_options=() _builder_deps=() # array of all dependencies for this script _builder_options_inheritable=() # array of all options that should be passed to child scripts _builder_default_action=build declare -A -g _builder_targets_excluded_by_platform=() declare -A -g _builder_params declare -A -g _builder_options_short declare -A -g _builder_options_var declare -A -g _builder_dep_path # array of output files for action:target pairs declare -A -g _builder_dep_related_actions # array of action:targets associated with a given dependency declare -A -g _builder_internal_dep # array of internal action:targets dependency relationships declare -A -g _builder_target_paths # array of target child project paths declare -A -g _builder_dep_targets # array of :targets given for a specific dependency (comma separated if more than one) shift local sub=() # describe each target, action, and option possibility while [[ $# -gt 0 ]]; do local key="$1" local value="$key" local description= if [[ $key =~ [[:space:]] ]]; then IFS=" " read -r -a sub <<< "$key" value="${sub[0]}" description="$(builder_trim "${sub[@]:1}")" fi local original_value="$value" if [[ $value =~ ^: ]]; then # Parameter is a target local target_path= if [[ $value =~ = ]]; then # The target has a custom child project path IFS="=" read -r -a sub <<< "$value" target_path="${sub[@]:1}" value="${sub[0]}" original_value="$value" if [[ ! -d "$THIS_SCRIPT_PATH/$target_path" ]]; then builder_die "Target path '$target_path' for $value does not exist." fi else # If the target name matches a folder name, implicitly # make it available as a child project if [[ -d "$THIS_SCRIPT_PATH/$_builder_child_base${value:1}" ]]; then target_path="$_builder_child_base${value:1}" fi fi _builder_targets+=($value) if [[ ! -z "$target_path" ]]; then _builder_target_paths[$value]="$target_path" fi elif [[ $value =~ ^@ ]]; then # Parameter is a dependency local dependency="${value:1}" local dependency_target= # all targets if [[ $dependency =~ : ]]; then IFS=":" read -r dependency dependency_target <<< "$dependency" dependency_target=":$dependency_target" fi dependency="`_builder_expand_relative_path "$dependency"`" _builder_deps+=($dependency) _builder_dep_related_actions[$dependency]="$(_builder_expand_action_targets "$description")" _builder_dep_targets[$dependency]="$dependency_target" # We don't want to add deps to params, so shift+continue shift continue elif [[ $value =~ ^-- ]]; then # Parameter is an option # Look for a shorthand version of the option local option_var= if [[ $value =~ = ]]; then IFS="=" read -r -a sub <<< "$value" option_var="${sub[@]:1}" value="${sub[0]}" fi local is_inheritable=false if [[ $value =~ \+$ ]]; then # final + indicates that option is inheritable is_inheritable=true value="${value:0:-1}" fi if [[ $value =~ , ]]; then IFS="," read -r -a sub <<< "$value" local option_long="${sub[0]}" local option_short="${sub[@]:1}" _builder_options+=($option_long) if $is_inheritable; then _builder_options_inheritable+=($option_long) fi _builder_options_short[$option_short]="$option_long" if [[ ! -z "$option_var" ]]; then _builder_options_var[$option_long]="$option_var" fi value="$option_long, $option_short" else _builder_options+=($value) if $is_inheritable; then _builder_options_inheritable+=($value) fi if [[ ! -z "$option_var" ]]; then _builder_options_var[$value]="$option_var" fi fi if [[ ! -z $option_var ]]; then value="$value $option_var" fi else # Parameter is an action if [[ $value =~ \+$ ]]; then # If the action name has a '+' suffix then it is the default action value=${value//+} _builder_default_action=$value fi _builder_actions+=($value) fi if [[ -z "${description}" ]]; then description=$(_builder_get_default_description "$original_value") fi _builder_params[${original_value}]="$description" shift done # We'll always add a :project if no target is specified if (( ! ${#_builder_targets[@]} )); then _builder_targets+=(:project) _builder_params[\:project]=$(_builder_get_default_description ":project") fi } # # Defines an output file or folder expected to be present after successful # completion of an action for a target. Used to skip actions for dependency # builds. If `:target` is not provided, assumes `:project`. # # Relative paths are relative to script folder; absolute paths are relative # to repository root, not filesystem root. # # ### Usage # # ```bash # builder_describe_outputs action:target filename [...] # ``` # # ### Parameters # # * 1: `action[:target]` action and/or target associated with file # * 2: `filename` name of file or folder to check # * 3+: ... repeat previous arguments for additional outputs # # ### Example # # ```bash # builder_describe_outputs \ # configure /node_modules \ # build build/index.js # ``` # function builder_describe_outputs() { _builder_record_function_call builder_describe_outputs while [[ $# -gt 0 ]]; do local key="$1" path="$2" action= target= path="`_builder_expand_relative_path "$path"`" if [[ $key =~ : ]]; then IFS=":" read -r action target <<< "$key" target=":$target" else # Add dependency expected output file for all targets, as well as a # wildcard target match action="$key" for target in "${_builder_targets[@]}"; do _builder_dep_path[$action$target]="$path" done target=':*' fi _builder_dep_path[$action$target]="$path" shift 2 done _builder_define_default_internal_dependencies # We only want to define internal dependencies after both builder_parse and builder_describe_outputs have been called if _builder_has_function_been_called builder_parse; then _builder_add_chosen_action_target_dependencies fi } _builder_get_default_description() { local description= local value="$1" # default descriptions for common build actions, targets, and options case "$value" in clean) description="remove build/ folder and build artifacts" ;; configure) description="install dependencies, e.g. npm" ;; build) description="build target(s)" ;; test) description="run automated tests" ;; :project) description="this project" ;; :app) description="main app" ;; :engine) description="engine module" ;; :module) description="this module" ;; :tools) description="build tools for this project" ;; --timings) description="report on timings" ;; --debug) description="debug build" ;; --release) description="release build (prevents --debug in local-env builds)" ;; esac echo "$description" } _builder_parameter_error() { local program="$1" local type="$2" local param="$3" builder_echo red "$program: invalid $type: $param" echo builder_display_usage exit 64 } # # Pre-initializes the color setting based on the options specified to a # a build.sh script. This is called automatically during init. # # Parameters # 1: "$@" all command-line arguments # _builder_check_color() { # Process command-line arguments while [[ $# -gt 0 ]] ; do local key="$1" case "$key" in --color) builder_use_color true ;; --no-color) builder_use_color false ;; esac shift # past the processed argument done } # # For every build action:target in _builder_chosen_action_targets, add # its full internal dependency tree # _builder_add_chosen_action_target_dependencies() { local action_target i=0 new_actions=() # Iterate through every action specified on command line; we use this loop # style so that any new actions added here will also be iteratively checked while (( $i < ${#_builder_chosen_action_targets[@]} )); do action_target=${_builder_chosen_action_targets[$i]} # If we have an internal dependency for the chosen action:target pair if [[ ! -z ${_builder_internal_dep[$action_target]+x} ]]; then local dep_outputs=(${_builder_internal_dep[$action_target]}) dep_output for dep_output in "${dep_outputs[@]}"; do # If there is a defined output for this dependency if [[ ! -z ${_builder_dep_path[$dep_output]+x} ]]; then # If the output for the dependency is missing, or we have --force-deps if [[ ! -e "$REPO_ROOT/${_builder_dep_path[$dep_output]}" ]] || builder_is_full_dep_build; then # Add the dependency to the chosen action:target list if ! _builder_item_in_array "$dep_output" "${_builder_chosen_action_targets[@]}"; then _builder_chosen_action_targets+=($dep_output) new_actions+=($dep_output) fi fi fi done fi i=$((i + 1)) done if [[ ${#new_actions[@]} -gt 0 ]]; then if builder_is_full_dep_build; then builder_echo "Automatically running all dependency actions due to --force-deps:" else builder_echo "Automatically running following required actions with missing outputs:" fi for e in "${new_actions[@]}"; do builder_echo "* $e" done fi } # # If we have described outputs, then we will setup our # default internal dependency chain: # # configure <- build <- (test,install,publish) # _builder_define_default_internal_dependencies() { for target in "${_builder_targets[@]}"; do _builder_define_default_internal_deps_for_target "$target" done _builder_define_default_internal_deps_for_target ':*' } _builder_define_default_internal_deps_for_target() { local target=$1 _builder_define_default_internal_dep "$target" configure build _builder_define_default_internal_dep "$target" build test _builder_define_default_internal_dep "$target" build install } _builder_define_default_internal_dep() { local target=$1 dep=$2 action=$3 if _builder_item_in_array $dep "${_builder_actions[@]}" && _builder_item_in_array $action "${_builder_actions[@]}"; then [[ -z ${_builder_internal_dep[$action$target]+x} ]] && _builder_internal_dep[$action$target]=$dep$target || _builder_internal_dep[$action$target]="${_builder_internal_dep[$action$target]} $dep$target" fi } # # Define a local dependency between one action:target and # another. # # Usage: # builder_describe_internal_dependency action:target depaction:deptarget ... # Parameters: # 1: action:target The action and target that has a dependency # 2: depaction:deptarget The dependency action and target # Example: # builder_describe_internal_dependency \ # build:mac build:mac-x86_64 \ # build:mac build:mac-arm64 # # Note: actions and targets must be fully specified, and this _must_ be called # before both of builder_describe_outputs and builder_parse in order for # dependencies to be resolved. builder_describe_internal_dependency() { _builder_record_function_call builder_describe_internal_dependency if _builder_has_function_been_called builder_parse && _builder_has_function_been_called builder_describe_outputs; then builder_warn "WARNING: builder_describe_internal_dependency needs to be called before builder_parse and builder_describe_outputs have both been called" fi while [[ $# -gt 0 ]]; do local action_target=$1 dep_action_target=$2 [[ -z ${_builder_internal_dep[$action_target]+x} ]] && _builder_internal_dep[$action_target]=$dep_action_target || _builder_internal_dep[$action_target]="${_builder_internal_dep[$action_target]} $dep_action_target" shift 2 done } # Initializes a build.sh script, parses command line. Will abort the script if # invalid parameters are passed in. Use together with builder_describe which # sets up the possible command line parameters # # Usage: # builder_parse "$@" # Parameters # 1: $@ command-line arguments builder_parse() { _builder_record_function_call builder_parse local exp=() builder_extra_params=() while [[ $# -gt 0 ]] ; do local action= target= local key="$1" if [[ $key == "--" ]]; then shift builder_extra_params=("$@") break fi if [[ $key =~ ^- ]]; then # Expand short -o to --option in options lookup if [[ ! -z ${_builder_options_short[$key]+x} ]]; then key=${_builder_options_short[$key]} fi exp+=($key) if [[ ! -z ${_builder_options_var[$key]+x} ]]; then shift if [[ $# -eq 0 ]]; then _builder_parameter_error "$0" parameter "$key" fi exp+=("$1") fi else # Expand comma separated values if [[ $key =~ : ]]; then IFS=: read -r action target <<< "$key" else action="$key" target= fi local actions targets IFS=, read -r -a actions <<< "$action" IFS=, read -r -a targets <<< "$target" if [[ "${#actions[@]}" -eq 0 ]]; then # No actions, so must be at least one :target for target in "${targets[@]}"; do exp+=(:$target) done else for action in "${actions[@]}"; do if [[ "${#targets[@]}" -eq 0 ]]; then # No :targets so just expand actions exp+=($action) else # Actions:targets, expand them all for target in "${targets[@]}"; do exp+=($action:$target) done fi done fi fi shift done _builder_parse_expanded_parameters "${exp[@]}" } _builder_parse_expanded_parameters() { _builder_build_deps=--deps builder_verbose= if [[ -z "${_builder_timings+x}" ]]; then # Only set _builder_timings if not already set if builder_is_ci_build; then _builder_timings=true else _builder_timings=false fi export _builder_timings fi builder_debug= local is_release=false local _params=($@) _builder_chosen_action_targets=() _builder_chosen_options=() _builder_current_action= _builder_is_child=1 _builder_offline= _builder_ignore_unknown_options=1 builder_ignored_options=() local n=0 # Process command-line arguments while [[ $# -gt 0 ]] ; do local key="$1" local action= local target= local e has_action has_target has_option longhand_option if [[ $key =~ : ]]; then IFS=: read -r action target <<< "$key" target=:$target else action="$key" target= fi # Expand shorthand parameters new_action=$(_builder_expand_shorthand $action "${_builder_actions[@]}") || case $? in 1) action=$new_action ;; *) builder_warn "ERROR: Parameter $action has $? matches, could mean any of {$new_action}" exit 1 ;; esac new_target=$(_builder_expand_shorthand $target "${_builder_targets[@]}") || case $? in 1) target=$new_target ;; *) builder_warn "ERROR: Parameter $target has $? matches, could mean any of {$new_target}" exit 1 ;; esac _builder_item_in_array "$action" "${_builder_actions[@]}" && has_action=1 || has_action=0 _builder_item_in_array "$target" "${_builder_targets[@]}" && has_target=1 || has_target=0 if (( has_action )) || (( has_target )); then # Document parameter expansion for end use _params[$n]=$action$target fi n=$((n + 1)) _builder_item_in_array "$key" "${_builder_options[@]}" && has_option=1 || has_option=0 if (( has_action )) && (( has_target )); then # apply the selected action and selected target _builder_chosen_action_targets+=("$action$target") elif (( has_action )); then # apply the selected action to all targets if [[ ! -z $target ]]; then # A target was specified but is not valid if builder_is_target_excluded_by_platform "$target"; then builder_echo red "$0: Target $target is unavailable because it requires ${_builder_targets_excluded_by_platform[$target]}" echo builder_display_usage exit 64 else _builder_parameter_error "$0" target "$target" fi fi for e in "${_builder_targets[@]}"; do _builder_chosen_action_targets+=("$action$e") done elif (( has_target )); then # apply the default action to the selected target if [[ ! -z $action ]]; then # An action was specified but is not valid _builder_parameter_error "$0" action "$action" fi _builder_chosen_action_targets+=("$_builder_default_action$target") elif (( has_option )); then _builder_chosen_options+=("$key") if [[ ! -z ${_builder_options_var[$key]+x} ]]; then shift n=$((n + 1)) # Set the variable associated with this option to the next parameter value # A little bit of hoop jumping here to avoid issues with cygwin paths being # corrupted too early in the game local varname=${_builder_options_var[$key]} declare -g $varname="$1" fi else # See also _builder_reserved_parameters case "$key" in --help|-h) builder_display_usage exit 0 ;; --color) builder_use_color true ;; --no-color) builder_use_color false ;; --verbose|-v) _builder_chosen_options+=(--verbose) builder_verbose=--verbose ;; --timings) _builder_timings=true ;; --no-timings) _builder_timings=false ;; --debug) _builder_chosen_options+=(--debug) builder_debug=--debug ;; --release) _builder_chosen_options+=(--release) # As of #13827, this is only checked when detecting if --debug should be auto-applied. is_release=true ;; --deps|--no-deps|--force-deps) _builder_build_deps=$key ;; --builder-dep-parent) # internal use parameter for dependency builds - identifier of parent script shift builder_dep_parent="$1" builder_echo setmark "dependency build, started by $builder_dep_parent" builder_echo grey "$(basename "$0") parameters: <${_params[@]}>" ;; --builder-child) _builder_is_child=0 builder_echo setmark "child build, parameters: <${_params[@]}>" ;; --builder-report-dependencies) # internal reporting function, ignores all other parameters _builder_report_dependencies ;; --builder-completion-describe) _builder_completion_describe exit 0 ;; --offline) _builder_offline=--offline ;; --builder-ignore-unknown-options) _builder_ignore_unknown_options=0 ;; *) # script does not recognize anything of action or target form at this point. if [[ $key =~ ^: ]]; then # This is an unsupported target if builder_is_target_excluded_by_platform "$key"; then builder_echo red "$0: Target $key is unavailable because it requires ${_builder_targets_excluded_by_platform[$key]}" echo builder_display_usage exit 64 else _builder_parameter_error "$0" target "$key" fi elif builder_is_child_build; then # For child builds, don't fail the build when pass inheritable # parameters (#11408) builder_echo_debug "Parameter '$key' is not supported, ignoring" elif [[ $key =~ ^- ]] && builder_ignore_unknown_options; then builder_echo warning "Ignoring unknown option $key" builder_ignored_options+=("$key") else _builder_parameter_error "$0" parameter "$key" fi esac fi shift # past the processed argument done # Add default action if not specified, but not for child builds if (( ! ${#_builder_chosen_action_targets[@]} )); then if ! builder_is_child_build; then for e in "${_builder_targets[@]}"; do _builder_chosen_action_targets+=("$_builder_default_action$e") done fi fi # We only want to define internal dependencies after both builder_parse and builder_describe_outputs have been called if _builder_has_function_been_called builder_describe_outputs; then _builder_add_chosen_action_target_dependencies fi if $_builder_debug_internal; then builder_echo_debug "Selected actions and targets:" for e in "${_builder_chosen_action_targets[@]}"; do builder_echo_debug "* $e" done builder_echo_debug builder_echo_debug "Selected options:" for e in "${_builder_chosen_options[@]}"; do builder_echo_debug "* $e" done fi if builder_is_dep_build; then _builder_verify_expected_sub_process_variables elif builder_is_child_build; then _builder_verify_expected_sub_process_variables else # For now, we'll leave this warning disabled, but may re-enable in #15130 # if [[ ! -z "${_builder_deps_built+x}" ]]; then # builder_warn "WARNING: Child build '${THIS_SCRIPT_IDENTIFIER}' instantiated without using builder_launch" # fi # This is a top-level invocation, so we want to track which dependencies # have been built, so they don't get built multiple times. export _builder_deps_built=`mktemp` # We will also track timings _builder_timing_setup # Per #11106, local builds use --debug by default. # Second condition prevents the block (and message) from executing when --debug is already specified explicitly. if [[ ${KEYMAN_VERSION_ENVIRONMENT:-} == "local" ]] && [[ $builder_debug != --debug ]] && ! $is_release; then builder_echo grey "Local build environment detected: setting --debug" _params+=(--debug) _builder_chosen_options+=(--debug) builder_debug=--debug fi builder_echo setmark "$(basename "$0") parameters: <${_params[@]}>" if [[ ${#builder_extra_params[@]} -gt 0 ]]; then builder_echo grey "$(basename "$0") extra parameters: <${builder_extra_params[@]}>" fi fi if builder_is_debug_build; then BUILDER_CONFIGURATION=debug else BUILDER_CONFIGURATION=release fi # Now that we've successfully parsed options adhering to the _builder spec, we may activate our # action_failure and action_hanging traps. (We don't want them active on scripts not yet using # said script.) # # Note: if an error occurs within a script's function in a `set -e` script, it becomes an exit # instead for the function's caller. So, we need both `err` and `exit` here. # See https://medium.com/@dirk.avery/the-bash-trap-trap-ce6083f36700. if [[ -z "$(trap -p DEBUG)" ]]; then # not running in bashdb trap _builder_failure_trap err exit fi } _builder_verify_expected_sub_process_variables() { if [[ -z ${_builder_deps_built+x} ]]; then builder_die "FATAL ERROR: Expected '_builder_deps_built' variable to be set" fi if $_builder_timings; then if [[ -z ${_builder_timing_report_file+x} ]]; then builder_die "FATAL ERROR: Expected '_builder_timing_report_file' variable to be set" fi fi } # # Returns 0 (true) if --builder-ignore-unknown-options flag has been specified # in the command line # function builder_ignore_unknown_options() { return $_builder_ignore_unknown_options } _builder_pad() { local count=$1 local text1=$2 local text2=$3 local fmt="%-${count}s%s\n" printf $fmt "$text1" "$text2" } _builder_completion_describe() { printf '%s ' "${_builder_actions[@]}" echo -n "; " printf '%s ' "${_builder_targets[@]}" echo -n "; " local opts=("${_builder_options[@]}") # Add default options; see also _builder_reserved_parameters opts+=( --verbose --debug --release --color --no-color --offline --help --timings --no-timings ) if builder_has_dependencies; then opts+=( --deps --no-deps --force-deps ) fi printf '%s ' "${opts[@]}" } builder_display_usage() { local e program description # Minimum padding is 12 characters, increase this if necessary # if you add other, longer, global options (like --verbose, --debug) local width=12 for e in "${!_builder_params[@]}"; do if (( ${#e} > $width )); then width=${#e} fi done width=$((width + 6)) program="$(basename "$0")" if [[ ! -z ${_builder_description+x} ]]; then echo "Summary:" echo " $_builder_description" echo fi echo "Script Identifier:" echo " $THIS_SCRIPT_IDENTIFIER" echo echo "Usage:" echo " $program [options...] [action][:target]..." echo echo "Actions: " for e in "${_builder_actions[@]}"; do if [[ ! -z "${_builder_params[$e]+x}" ]]; then description="${_builder_params[$e]}" else description=$(_builder_get_default_description "$e") fi _builder_pad $width " $e" "$description" done echo echo "Targets: " for e in "${_builder_targets[@]}"; do if [[ ! -z "${_builder_params[$e]+x}" ]]; then description="${_builder_params[$e]}" else description=$(_builder_get_default_description "$e") fi _builder_pad $width " $e" "$description" done if [[ ${#_builder_targets_excluded_by_platform[@]} -gt 0 ]]; then echo echo "Unavailable Targets: " for e in "${!_builder_targets_excluded_by_platform[@]}"; do if [[ ! -z "${_builder_params[$e]+x}" ]]; then description="${_builder_params[$e]}" else description=$(_builder_get_default_description "$e") fi _builder_pad $width " $e" "$description (requires ${_builder_targets_excluded_by_platform[$e]})" done fi echo echo "Options: " for e in "${!_builder_params[@]}"; do if [[ $e =~ ^-- ]]; then _builder_pad $width " $e" "${_builder_params[$e]}" fi done # See also _builder_reserved_parameters _builder_pad $width " --verbose, -v" "Verbose logging" _builder_pad $width " --debug, -d" "Debug build" _builder_pad $width " --color" "Force colorized output" _builder_pad $width " --no-color" "Never use colorized output" _builder_pad $width " --offline" "Try to build while being offline" if builder_has_dependencies; then _builder_pad $width " --deps" "Build dependencies if required (default)" _builder_pad $width " --no-deps" "Skip build of dependencies" _builder_pad $width " --force-deps" "Reconfigure and rebuild all dependencies" fi _builder_pad $width " --help, -h" "Show this help" echo echo "Dependencies: " if builder_has_dependencies; then for d in "${_builder_deps[@]}"; do echo " $d" done else echo " This module has no dependencies" fi # Defined in `builder_use_color`; this assumes that said func has been called. local c1=$BUILDER_TERM_START local c0=$BUILDER_TERM_END echo echo -e "* Specify ${c1}action:target${c0} to run a specific ${c1}action${c0} against a specific ${c1}:target${c0}." echo -e "* If ${c1}action${c0} is specified without a ${c1}target${c0} suffix, it will be applied to all ${c1}:target${c0}s." echo -e "* If ${c1}:target${c0} is specified without an ${c1}action${c0} prefix, ${c1}$_builder_default_action:target${c0} will be inferred." echo -e "* If no ${c1}action${c0}, ${c1}:target${c0}, or ${c1}action:target${c0} entries are specified, ${c1}$_builder_default_action${c0} will run on all ${c1}:target${c0}s." echo } builder_finish_action() { local result="$1" local action="$2" target action_name if [[ $action =~ : ]]; then IFS=: read -r action target <<< "$action" target=":$target" else target=':*' fi if [[ "$target" == ":*" ]]; then action_name="$action" else action_name="$action$target" fi local matched_action="$action$target" if [[ "$matched_action" == "${_builder_current_action}" ]]; then if [[ $result == success ]]; then # Sanity check: if there is a described output for this action, does the corresponding # file or directory exist now? if _builder_dep_output_defined $matched_action && ! _builder_dep_output_exists "$matched_action"; then builder_echo warning "Expected output: '${_builder_dep_path[$matched_action]}'." builder_echo end "$action_name" warning "## $action_name completed successfully, but output does not exist" else builder_echo end "$action_name" success "## $action_name completed successfully" fi elif [[ $result == failure ]]; then builder_echo end "$action_name" error "## $action_name failed" else builder_echo end "$action_name" error "## $action_name failed with message: $result" fi _builder_timing_stop "${_builder_matched_action}" # Remove $action$target from the array; it is no longer a current action _builder_current_action= else builder_echo warning "reporting result of $action_name but the action was never started!" fi } #------------------------------------------------------------------------------------------ # Dependencies #------------------------------------------------------------------------------------------ _builder_dep_output_defined() { if [[ ! -z ${_builder_dep_path[$1]+x} ]]; then return 0 else return 1 fi } _builder_dep_output_exists() { if _builder_dep_output_defined $1 && [[ -e "$REPO_ROOT/${_builder_dep_path[$1]}" ]]; then return 0 else return 1 fi } # # Returns `0` if the dependency should be built for the given action:target # _builder_should_build_dep() { local action_target="$1" local dep="$2" local related_actions=(${_builder_dep_related_actions[$dep]}) if [[ $action_target =~ ^clean ]]; then # don't attempt to build dependencies for a 'clean' action return 1 fi if ! _builder_item_in_glob_array "$action_target" "${related_actions[@]}"; then return 1 fi return 0 } # # Removes a dependency from the list of available dependencies # # Parameters: # $1 path to dependency # builder_remove_dep() { local dependency="$1" i dependency="`_builder_expand_relative_path "$dependency"`" for i in "${!_builder_deps[@]}"; do if [[ ${_builder_deps[i]} = $dependency ]]; then unset '_builder_deps[i]' fi done # rebuild the array to remove the empty item _builder_deps=( "${_builder_deps[@]}" ) } # # Configure and build all dependencies # Later, may restrict by either action or target # _builder_do_build_deps() { local action_target="$1" if [[ $_builder_build_deps == --no-deps ]]; then # we've been asked to skip dependencies return 0 fi _builder_timing_start "$1:dependencies" for dep in "${_builder_deps[@]}"; do # Don't attempt to build dependencies that don't match the current # action:target (wildcards supported for matches here) if ! _builder_should_build_dep "$action_target" "$dep"; then builder_echo "Skipping dependency $dep for $_builder_matched_action_name" continue fi dep_target= if [[ ! -z ${_builder_dep_targets[$dep]+x} ]]; then # TODO: in the future split _builder_dep_targets into comma-separated # array for multiple targets for a dep? dep_target=${_builder_dep_targets[$dep]} fi # Only configure and build the dependency once per invocation if builder_has_module_been_built "$dep$dep_target"; then continue fi builder_set_module_has_been_built "$dep$dep_target" "$REPO_ROOT/$dep/build.sh" "configure$dep_target" "build$dep_target" \ $builder_verbose \ $builder_debug \ $_builder_offline \ $_builder_build_deps \ --builder-dep-parent "$THIS_SCRIPT_IDENTIFIER" && ( if $_builder_debug_internal; then builder_echo success "## Dependency $dep$dep_target for $_builder_matched_action_name completed successfully" fi ) || ( result=$? builder_echo error "## Dependency failed with exit code $result" exit $result ) || exit $? # Required due to above subshell masking exit done _builder_timing_stop "$1:dependencies" } # # returns `0` if we are in a dependency doing a build. # builder_is_dep_build() { if [[ ! -z ${builder_dep_parent+x} ]]; then return 0 fi return 1 } # # returns `0` if we are in a child script doing a build # builder_is_child_build() { return $_builder_is_child } # # return `1` for a regular build, `0` to try to build while being offline. # The offline build might fail if not all dependencies are cached. # builder_try_offline() { if [[ "${_builder_offline}" == "--offline" ]]; then return 0 else return 1 fi } # # returns `0` if we should attempt to do quick builds in a dependency build, for # example skipping `tsc -b` where a parent may also do it; corresponds to the # `--deps` parameter (which is the default). # builder_is_quick_dep_build() { if [[ $_builder_build_deps == --deps ]]; then return 0 fi return 1 } # # returns `0` if we should do a full configure and build in a dependency build; # corresponds to the `--force-deps`` parameter. # builder_is_full_dep_build() { if [[ $_builder_build_deps == --force-deps ]]; then return 0 fi return 1 } # # returns `0` if the current build script has at least one dependency. # builder_has_dependencies() { if [[ ${#_builder_deps[@]} -eq 0 ]]; then return 1 fi return 0 } # # Tests if a dependency module has been built already in the current script # invocation; if not running in a builder context, always returns `1` (i.e. # "false"). # # ### Usage # # ```bash # builder_has_module_been_built dependency-name # ``` # # ### Parameters # # * 1: `dependency-name` the `$SCRIPT_IDENTIFIER` of the dependency # (repo-relative path without leading `/`); or for # external dependencies, a path-like starting with # `/external/`. # # ### Examples # # ```bash # if builder_has_module_been_built common/web/keyman-version; then ... # if builder_has_module_been_built /external/npm-ci; then ... # ``` # builder_has_module_been_built() { local module="$1" if [[ -z ${_builder_deps_built+x} ]]; then # not in a builder context, so we assume a build is needed return 1 fi if [[ -f $_builder_deps_built ]] && grep -qx "$module" $_builder_deps_built; then # dependency history file contains the dependency module return 0 fi return 1 } # # Updates the dependency module build state for the current script invocation; # if not running in a builder context, a no-op. # # ### Usage # # ```bash # builder_set_module_has_been_built dependency-name # ``` # # ### Parameters # # * 1: `dependency-name` the `$SCRIPT_IDENTIFIER` of the dependency # (repo-relative path without leading `/`); or for # external dependencies, a path-like starting with # `/external/`. # # ### Examples # # ```bash # builder_set_module_has_been_built common/web/keyman-version # builder_set_module_has_been_built /external/npm-ci # ``` # builder_set_module_has_been_built() { local module="$1" if [[ ! -z ${_builder_deps_built+x} ]]; then echo "$module" >> $_builder_deps_built fi } # # Reports on all described dependencies, then exits # used by builder-controls.sh # _builder_report_dependencies() { echo "${_builder_deps[@]}" exit 0 } #------------------------------------------------------------------------------------------ # Utility functions #------------------------------------------------------------------------------------------ # # returns `0` if we should be verbose in output # builder_verbose() { if [[ $builder_verbose == --verbose ]]; then return 0 fi return 1 } # # returns `0` if we are doing a debug build # builder_is_debug_build() { if [[ $builder_debug == --debug ]]; then return 0 fi return 1 } # # Track whether functions have already been called; # later we may use this to prevent multiple calls to, e.g. # builder_describe # _builder_function_calls=() _builder_record_function_call() { local func=$1 if _builder_has_function_been_called $1; then # builder_die "ERROR: $func cannot be called more than once." return 0 fi _builder_function_calls+=($1) } _builder_has_function_been_called() { local func=$1 if _builder_item_in_array $1 "${_builder_function_calls[@]}"; then return 0 fi return 1 } ################################################################################ # Platform restrictions ################################################################################ # Describes the platforms for which a given target will be available, and # filters the list of targets accordingly, removing targets that cannot be built # on the current platform. Removed targets will be listed in a separate section # in the help documentation. # # Parameters: target platform[s] ... Target and its list of corresponding # platform(s) # # Multiple targets can be listed. Each target must be followed by a comma # separated list of platforms. The currently supported platforms are: # # Operating System platforms: # win Windows # mac macOS # linux Linux # # Development tooling platforms: # delphi Delphi is installed (on Windows only) # android Android Studio is installed # builder_describe_platform() { local builder_platforms=(linux mac win) local builder_tools=(android-studio delphi) local builder_platform="${BUILDER_OS}" # --- Detect tools --- local builder_installed_tools=() # Detect delphi compiler (see also delphi_environment.inc.sh) if builder_is_windows; then local ProgramFilesx86="$(cygpath -w -F 42)" if [[ -x "$(cygpath -u "$ProgramFilesx86\\Embarcadero\\Studio\\20.0\\bin\\dcc32.exe")" ]]; then builder_installed_tools+=(delphi) fi fi # Detect Android Studio based on ANDROID_HOME environment variable if [[ ! -z ${ANDROID_HOME+x} ]]; then builder_installed_tools+=(android-studio) fi # --- Overrides --- # For testing, we can override the current platform if [[ ! -z "${BUILDER_PLATFORM_OVERRIDE+x}" ]]; then builder_warn "WARNING: BUILDER_PLATFORM_OVERRIDE variable found. Overriding detected platform '$builder_platform' with '$BUILDER_PLATFORM_OVERRIDE'" builder_platform="$BUILDER_PLATFORM_OVERRIDE" fi # For testing, we can override the current tools if [[ ! -z "${BUILDER_TOOLS_OVERRIDE+x}" ]]; then builder_warn "WARNING: BUILDER_TOOLS_OVERRIDE variable found. Overriding detected tools (${builder_installed_tools[@]}) with (${BUILDER_TOOLS_OVERRIDE[@]})" builder_installed_tools=("${BUILDER_TOOLS_OVERRIDE[@]}") fi # --- Exclude targets depending on missing tools and platforms while [[ $# -gt 1 ]]; do local target="$1" shift local platforms platform IFS="," read -r -a platforms <<< "$1" shift for platform in "${platforms[@]}"; do if _builder_item_in_array "$platform" "${builder_platforms[@]}"; then # Check operating system if [[ "$platform" != "$builder_platform" ]]; then _builder_targets_excluded_by_platform[$target]=$platform _builder_targets=( ${_builder_targets[@]/$target} ) fi elif _builder_item_in_array "$platform" "${builder_tools[@]}"; then # Check installed tools if ! _builder_item_in_array "$platform" "${builder_installed_tools[@]}"; then _builder_targets_excluded_by_platform[$target]=$platform _builder_targets=( ${_builder_targets[@]/$target} ) fi else builder_warn "Available platforms: ${builder_platforms[@]}" builder_warn "Available tools: ${builder_tools[@]}" builder_die "Invalid platform or tool specified for builder_describe_platforms: $platform" fi done done if [[ ${#_builder_targets[@]} == 0 ]]; then builder_die "This script cannot be run on $builder_platform; supported platforms are: ${_builder_targets_excluded_by_platform[@]}" fi } # Returns 0 if the specified target is excluded by platform requirements # # Parameters: target Target to verify # builder_is_target_excluded_by_platform() { local target="$1" if [[ ! -z "${_builder_targets_excluded_by_platform[$target]+x}" ]]; then return 0 fi return 1 } # Returns 0 if the script is running in a Docker container builder_is_running_on_docker() { if [[ -z ${DOCKER_RUNNING:-} ]]; then return 1 else return 0 fi } # Returns 0 if the script is running in a GitHub Actions environment builder_is_running_on_gha() { if [[ -z ${GITHUB_ACTIONS:-} ]]; then return 1 else return 0 fi } # Returns 0 if the script is running on TeamCity builder_is_running_on_teamcity() { if [[ -z "${TEAMCITY_GIT_PATH:-}" ]]; then return 1 else return 0 fi } # # Returns 0 if current build is running in CI, as a pull request test, or as a # mainline branch test, or as a release build # builder_is_ci_build() { if builder_is_ci_release_build || builder_is_ci_test_build; then return 0 fi return 1 } # # Returns 0 if current build is running as a release build in CI # builder_is_ci_release_build() { if [[ "${KEYMAN_VERSION_ENVIRONMENT:-}" =~ ^alpha|beta|stable$ ]]; then return 0 fi return 1 } # # Returns 0 if current build is running in CI, as a pull request test, or as a # mainline branch test # builder_is_ci_test_build() { if [[ "${KEYMAN_VERSION_ENVIRONMENT:-}" == test ]]; then return 0 fi return 1 } # # Returns 0 if current ci build is a release-level build. Do not use for non-ci # builds. # builder_is_ci_build_level_release() { if builder_is_ci_release_build; then return 0 fi if [[ "${KEYMAN_BUILD_LEVEL:-}" == release ]]; then return 0 fi return 1 } # # Returns 0 if current ci build is a build-level build. Do not use for non-ci # builds. # builder_is_ci_build_level_build() { if builder_is_ci_release_build; then return 1 fi if builder_is_ci_build_level_release; then # KEYMAN_BUILD_LEVEL == release, i.e. not build return 1 fi return 0 } # # Executes statement if a ci build level of 'release', and for local builds, but # not for a ci build level of 'build' # builder_if_release_build_level() { if builder_is_ci_build && builder_is_ci_build_level_build; then builder_echo "Skipping - buildLevel=build: $@" return 0 fi "$@" } # Returns 0 if we're running on Windows, i.e. if the environment variable # `OSTYPE` is set to "msys" or "cygwin". builder_is_windows() { if [[ "${OSTYPE:-}" == "msys" ]] || [[ "${OSTYPE:-}" == "cygwin" ]]; then return 0 else return 1 fi } # Returns 0 if we're running on macOS. builder_is_macos() { if [[ "${OSTYPE:-}" == darwin* ]]; then return 0 else return 1 fi } # Returns 0 if we're running on Linux (or rather if we're not running # on Windows or macOS). builder_is_linux() { if builder_is_windows || builder_is_macos; then return 1 else return 0 fi } # Sets the BUILDER_OS environment variable to linux|mac|win # _builder_get_operating_system() { declare -g BUILDER_OS if builder_is_macos; then BUILDER_OS=mac elif builder_is_windows; then BUILDER_OS=win else BUILDER_OS=linux fi readonly BUILDER_OS } ################################################################################ # Final initialization ################################################################################ # # Initialize builder once all functions are declared # _builder_init _builder_check_color "$@" # _builder_debug_internal flag can be used to emit verbose logs for builder itself, # e.g.: # _builder_debug_internal=true ./build.sh # if [ -z ${_builder_debug_internal+x} ]; then _builder_debug_internal=false fi if $_builder_debug_internal; then builder_echo_debug "Command line: $0 $@" fi