diff --git a/HISTORY.md b/HISTORY.md index ce6a887984..e14058d367 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,21 @@ # Keyman Version History +## 19.0.84 alpha 2025-07-15 + +* maint(common): add support for inline file exclusions to `add_zip_files` and unit tests (#14333) +* maint(developer): use `add_zip_files` function (#14334) + +## 19.0.83 alpha 2025-07-14 + +* maint(resources): add `build.sh` to run all `resource/build/test` tests (#14335) +* maint(developer): consolidate creating KMC_ZIP file (#14309) + +## 19.0.82 alpha 2025-07-10 + +* maint(android): fix Android release build (#14329) +* maint(mac): fix end of blocks (#14330) +* maint(mac): don't run `build.sh publish` twice (#14331) + ## 19.0.81 alpha 2025-07-09 * docs(developer): explanation of the Description field added in Details section. (#14304) diff --git a/VERSION.md b/VERSION.md index 88fec74280..70bf7a2baa 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -19.0.82 \ No newline at end of file +19.0.85 \ No newline at end of file diff --git a/developer/src/inst/build.sh b/developer/src/inst/build.sh index a8eb8b849f..07260c8b1d 100755 --- a/developer/src/inst/build.sh +++ b/developer/src/inst/build.sh @@ -20,9 +20,10 @@ builder_describe_outputs \ builder_parse "$@" -. "$KEYMAN_ROOT/resources/build/win/environment.inc.sh" -. "$KEYMAN_ROOT/resources/build/win/wix.inc.sh" -. "$KEYMAN_ROOT/resources/build/win/zip.inc.sh" +. "${KEYMAN_ROOT}/resources/build/win/environment.inc.sh" +. "${KEYMAN_ROOT}/resources/build/win/wix.inc.sh" +. "${KEYMAN_ROOT}/resources/build/win/zip.inc.sh" +. "${KEYMAN_ROOT}/resources/build/zip.inc.sh" # In dev environments, we'll hack the tier to alpha; CI sets this for us in real builds. if [[ -z ${KEYMAN_TIER+x} ]]; then @@ -242,27 +243,24 @@ function make-installer() { # } -# TODO: rename this to keyman-developer-cli-$Version.zip -KMC_ZIP="$DEVELOPER_ROOT/release/$KEYMAN_VERSION/kmcomp-$KEYMAN_VERSION.zip" - function make-kmc-install-zip() { builder_heading make-kmc-install-zip copy-schemas - cd "$DEVELOPER_ROOT/bin" + ( + # shellcheck disable=2164 + cd "${DEVELOPER_ROOT}/bin" - wzzip -bd -bb0 "$KMC_ZIP" \ - kmconvert.exe \ - sentry.dll sentry.x64.dll \ - kmdecomp.exe \ - keyboard_info.schema.json \ - kmp.schema.json \ - keyman-touch-layout.spec.json keyman-touch-layout.clean.spec.json \ - xml/layoutbuilder/*.keyman-touch-layout \ - projects/* \ - server/* + # TODO: rename this to keyman-developer-cli-$Version.zip + local KMCOMP_ZIP="${DEVELOPER_ROOT}/release/${KEYMAN_VERSION}/kmcomp-${KEYMAN_VERSION}.zip" - cd "$THIS_SCRIPT_PATH" + add_zip_files "${KMCOMP_ZIP}" -q -r \ + kmconvert.exe \ + keyboard_info.schema.json \ + xml/layoutbuilder/*.keyman-touch-layout \ + projects/ \ + server/ + ) } # TODO: are these required? diff --git a/resources/build/build-utils-ci.test.sh b/resources/build/build-utils-ci.test.sh index 662ea581da..8251156d27 100755 --- a/resources/build/build-utils-ci.test.sh +++ b/resources/build/build-utils-ci.test.sh @@ -18,7 +18,7 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" builder_echo "${COLOR_BLUE}## Testing: builder_pull_get_details with \$TEAMCITY_PR_NUMBER not set${COLOR_RESET}" if ! builder_pull_get_details; then - builder_echo "PASS: builder_pull_get_details should get no details if \$TEAMCITY_PR_NUMBER is not set" + builder_echo green " ✓ PASS: builder_pull_get_details should get no details if \$TEAMCITY_PR_NUMBER is not set" else builder_die "FAIL: builder_pull_get_details should get no details if \$TEAMCITY_PR_NUMBER is not set" fi @@ -27,7 +27,7 @@ fi builder_echo "${COLOR_BLUE}## Testing: builder_pull_get_details with \$TEAMCITY_PR_NUMBER=master${COLOR_RESET}" TEAMCITY_PR_NUMBER=master if ! builder_pull_get_details; then - builder_echo "PASS: builder_pull_get_details should get no details if \$TEAMCITY_PR_NUMBER=master" + builder_echo green " ✓ PASS: builder_pull_get_details should get no details if \$TEAMCITY_PR_NUMBER=master" else builder_die "FAIL: builder_pull_get_details should get no details if \$TEAMCITY_PR_NUMBER=master" fi @@ -44,7 +44,7 @@ builder_echo " pull number: $builder_pull_number" builder_echo " title: $builder_pull_title" builder_echo " labels: ${builder_pull_labels[*]}" -builder_echo "PASS: builder_pull_get_details with \$TEAMCITY_PR_NUMBER=$TEAMCITY_PR_NUMBER" +builder_echo green " ✓ PASS: builder_pull_get_details with \$TEAMCITY_PR_NUMBER=$TEAMCITY_PR_NUMBER" builder_echo "${COLOR_BLUE}## Testing: builder_pull_has_label epic-ldml${COLOR_RESET}" @@ -52,6 +52,6 @@ if ! builder_pull_has_label epic-ldml; then builder_die "FAIL: builder_pull_has_label epic-ldml" fi -builder_echo "PASS: builder_pull_has_label epic-ldml" +builder_echo green " ✓ PASS: builder_pull_has_label epic-ldml" builder_echo "All tests passed." diff --git a/resources/build/test/build.sh b/resources/build/test/build.sh new file mode 100755 index 0000000000..68aa96f461 --- /dev/null +++ b/resources/build/test/build.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +## START STANDARD BUILD SCRIPT INCLUDE +# adjust relative paths as necessary +THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" +. "${THIS_SCRIPT%/*}/../../../resources/build/builder.inc.sh" +## END STANDARD BUILD SCRIPT INCLUDE + +################################ Main script ################################ + +builder_describe \ + "Run unit tests (defined in *.test*.sh or test.sh) for builder scripts" \ + "test" + +builder_parse "$@" + +do_test() { + set +e + + # TODO: return non-0 exit code if any of the tests fail + while IFS= read -r -d '' file + do + builder_echo start "${file}" "Running tests for ${file}" + "${file}" + builder_echo end "${file}" success "Finished running tests for ${file}" + done < <(find . -name \*.test*.sh -o -name test.sh -print0) +} + +builder_run_action test do_test diff --git a/resources/build/test/builder-deps.test.sh b/resources/build/test/builder-deps.test.sh index 926142c973..7be2d6a37c 100755 --- a/resources/build/test/builder-deps.test.sh +++ b/resources/build/test/builder-deps.test.sh @@ -41,7 +41,7 @@ function test_dep_should_build() { if ! _builder_should_build_dep "$at" "resources/build/test/$dep"; then builder_die "FAIL: expecting to build dependency $dep for $at" else - echo "PASS: will build dependency $dep for $at" + builder_echo green " ✓ PASS: will build dependency $dep for $at" fi } @@ -52,7 +52,7 @@ function test_dep_should_not_build() { if _builder_should_build_dep "$at" "resources/build/test/$dep"; then builder_die "FAIL: not expecting to build dependency $dep for $at" else - echo "PASS: will not build dependency $dep for $at" + builder_echo green " ✓ PASS: will not build dependency $dep for $at" fi } @@ -119,7 +119,7 @@ test_dep_should_build test:project dep7 # is missing builder_parse test if [[ "${_builder_chosen_action_targets[@]}" == "test:project test:bar build:project build:bar" ]]; then - echo "PASS: 'build' actions automatically added" + builder_echo green " ✓ PASS: 'build' actions automatically added" else echo "All targets: ${_builder_chosen_action_targets[@]}" builder_die "FAIL: 'build' actions not automatically added, or unexpected action:targets added" diff --git a/resources/build/test/debug-deps/child/build.sh b/resources/build/test/debug-deps/child/build.sh index a21fed0709..6fc83b3fd9 100755 --- a/resources/build/test/debug-deps/child/build.sh +++ b/resources/build/test/debug-deps/child/build.sh @@ -12,10 +12,10 @@ builder_describe \ function do_build() { # Test the debug flag builder_is_debug_build || builder_die "FAIL: child: expecting builder_is_debug_build to be true" - echo "PASS: child: builder_is_debug_build is true" + builder_echo green " ✓ PASS: child: builder_is_debug_build is true" builder_has_option --debug || builder_die "FAIL: child: expecting builder_has_option --debug to be true" - echo "PASS: child: builder_has_option --debug is true" + builder_echo green " ✓ PASS: child: builder_has_option --debug is true" } builder_parse "$@" diff --git a/resources/build/test/debug-deps/child/dep/build.sh b/resources/build/test/debug-deps/child/dep/build.sh index 477e05603f..c5ad0216bb 100755 --- a/resources/build/test/debug-deps/child/dep/build.sh +++ b/resources/build/test/debug-deps/child/dep/build.sh @@ -12,10 +12,10 @@ builder_describe \ function do_build() { # Test the debug flag builder_is_debug_build || builder_die "FAIL: child/dep: expecting builder_is_debug_build to be true" - echo "PASS: child/dep: builder_is_debug_build is true" + builder_echo green " ✓ PASS: child/dep: builder_is_debug_build is true" builder_has_option --debug || builder_die "FAIL: child/dep: expecting builder_has_option --debug to be true" - echo "PASS: child/dep: builder_has_option --debug is true" + builder_echo green " ✓ PASS: child/dep: builder_has_option --debug is true" } builder_parse "$@" diff --git a/resources/build/test/debug-deps/dep/build.sh b/resources/build/test/debug-deps/dep/build.sh index a9ae70a28e..e94839b61a 100755 --- a/resources/build/test/debug-deps/dep/build.sh +++ b/resources/build/test/debug-deps/dep/build.sh @@ -13,10 +13,10 @@ builder_describe \ function do_build() { # Test the debug flag builder_is_debug_build || builder_die "FAIL: dep: expecting builder_is_debug_build to be true" - echo "PASS: dep: builder_is_debug_build is true" + builder_echo green " ✓ PASS: dep: builder_is_debug_build is true" builder_has_option --debug || builder_die "FAIL: dep: expecting builder_has_option --debug to be true" - echo "PASS: dep: builder_has_option --debug is true" + builder_echo green " ✓ PASS: dep: builder_has_option --debug is true" } builder_parse "$@" diff --git a/resources/build/test/debug-deps/dep/child/build.sh b/resources/build/test/debug-deps/dep/child/build.sh index 6537857373..a1b405f512 100755 --- a/resources/build/test/debug-deps/dep/child/build.sh +++ b/resources/build/test/debug-deps/dep/child/build.sh @@ -12,10 +12,10 @@ builder_describe \ function do_build() { # Test the debug flag builder_is_debug_build || builder_die "FAIL: child/dep: expecting builder_is_debug_build to be true" - echo "PASS: child/dep: builder_is_debug_build is true" + builder_echo green " ✓ PASS: child/dep: builder_is_debug_build is true" builder_has_option --debug || builder_die "FAIL: child/dep: expecting builder_has_option --debug to be true" - echo "PASS: child/dep: builder_has_option --debug is true" + builder_echo green " ✓ PASS: child/dep: builder_has_option --debug is true" } builder_parse "$@" diff --git a/resources/build/test/debug-deps/test.sh b/resources/build/test/debug-deps/test.sh index 9ac32d474a..5465c35e77 100755 --- a/resources/build/test/debug-deps/test.sh +++ b/resources/build/test/debug-deps/test.sh @@ -11,10 +11,10 @@ builder_describe \ function do_build() { builder_is_debug_build || builder_die "FAIL: parent: expecting builder_is_debug_build to be true" - echo "PASS: parent: builder_is_debug_build is true" + builder_echo green " ✓ PASS: parent: builder_is_debug_build is true" builder_has_option --debug || builder_die "FAIL: parent: expecting builder_has_option --debug to be true" - echo "PASS: parent: builder_has_option --debug is true" + builder_echo green " ✓ PASS: parent: builder_has_option --debug is true" } builder_parse build:child --debug diff --git a/resources/build/test/dependencies/test.sh b/resources/build/test/dependencies/test.sh index ec9c65bea7..059095b679 100755 --- a/resources/build/test/dependencies/test.sh +++ b/resources/build/test/dependencies/test.sh @@ -43,4 +43,4 @@ echo ------------------------------------------------------------------ fi ) || exit $? -echo "PASS: all dependency tests ran as expected" +builder_echo green " ✓ PASS: all dependency tests ran as expected" diff --git a/resources/build/test/test-utils.inc.sh b/resources/build/test/test-utils.inc.sh index a73f46773a..48569bedae 100644 --- a/resources/build/test/test-utils.inc.sh +++ b/resources/build/test/test-utils.inc.sh @@ -1,14 +1,20 @@ +# shellcheck shell=bash + function assert-equal() { local actual="$1" local expected="$2" local message= + local actual_to_show="'${actual}' " if [[ $# -gt 2 ]]; then message="$3: " fi + if [[ $# -gt 3 ]]; then + actual_to_show="" + fi - if [[ "$actual" != "$expected" ]]; then - builder_die " ✕ FAIL: ${message}actual result '$actual' should equal expected '$expected'" + if [[ "${actual}" != "${expected}" ]]; then + builder_die " ✕ FAIL: ${message}actual result '${actual}' should equal expected '${expected}'" else - builder_echo green " ✓ PASS: ${message}result '$actual' is correct" + builder_echo green " ✓ PASS: ${message}result ${actual_to_show}is correct" fi } diff --git a/resources/build/test/test.sh b/resources/build/test/test.sh index 9da63b09df..788af8c338 100755 --- a/resources/build/test/test.sh +++ b/resources/build/test/test.sh @@ -127,7 +127,7 @@ builder_describe \ builder_parse_test "clean:app test:engine" "--power" clean:app test:engine --power if builder_has_option --power; then - echo "PASS: --power option found" + builder_echo green " ✓ PASS: --power option found" else builder_die "FAIL: --power option not found" fi @@ -135,7 +135,7 @@ fi builder_parse_test "clean:app test:engine" "--zoom" clean:app test:engine -z if builder_has_option --zoom; then - echo "PASS: --zoom option found" + builder_echo green " ✓ PASS: --zoom option found" else builder_die "FAIL: --zoom option not found" fi @@ -147,7 +147,7 @@ function verify_option() { if builder_has_option ${OPTIONNAME}; then if [[ ${!VARIABLE} == ${EXPECTED} ]]; then - echo "PASS: ${OPTIONNAME} option variable \$${VARIABLE} has expected value '${EXPECTED}'" + builder_echo green " ✓ PASS: ${OPTIONNAME} option variable \$${VARIABLE} has expected value '${EXPECTED}'" else builder_die "FAIL: ${OPTIONNAME} option variable \$${VARIABLE} had value '${!VARIABLE}' but should have had '${EXPECTED}'" fi @@ -197,7 +197,7 @@ fi if [[ "${builder_ignored_options[@]}" != "--some-other-option" ]]; then builder_die "FAIL: --builder-ignore-unknown-options did not collect --some-other-option" fi -echo "PASS: --builder-ignore-unknown-options, --some-other-option was collected but ignored" +builder_echo green " ✓ PASS: --builder-ignore-unknown-options, --some-other-option was collected but ignored" # Test short form option with --builder-ignore-unknown-options builder_parse --builder-ignore-unknown-options -x @@ -207,21 +207,21 @@ fi if [[ "${builder_ignored_options[@]}" != "-x" ]]; then builder_die "FAIL: --builder-ignore-unknown-options '${builder_ignored_options[@]}' did not collect -x" fi -echo "PASS: --builder-ignore-unknown-options, -x was collected but ignored" +builder_echo green " ✓ PASS: --builder-ignore-unknown-options, -x was collected but ignored" # Test long form option without --builder-ignore-unknown-options (builder_parse --some-other-option >/dev/null) && EXIT_CODE=0 || EXIT_CODE=$? if [[ $EXIT_CODE != 64 ]]; then builder_die "FAIL: without --builder-ignore-unknown-options, --some-other-option parameter should not have been recognized" fi -echo "PASS: without --builder-ignore-unknown-options, --some-other-option was not a recognized parameter" +builder_echo green " ✓ PASS: without --builder-ignore-unknown-options, --some-other-option was not a recognized parameter" # Test short form option without --builder-ignore-unknown-options (builder_parse -x >/dev/null) && EXIT_CODE=0 || EXIT_CODE=$? if [[ $EXIT_CODE != 64 ]]; then builder_die "FAIL: without --builder-ignore-unknown-options, -x parameter should not have been recognized" fi -echo "PASS: without --builder-ignore-unknown-options, -x was not a recognized parameter" +builder_echo green " ✓ PASS: without --builder-ignore-unknown-options, -x was not a recognized parameter" # Test that ignored options variables are correct without --builder-ignore-unknown-options builder_parse test diff --git a/resources/build/test/trees/build.sh b/resources/build/test/trees/build.sh index 539b40f14d..8dad297009 100755 --- a/resources/build/test/trees/build.sh +++ b/resources/build/test/trees/build.sh @@ -24,7 +24,7 @@ builder_parse "$@" if builder_is_child_build; then builder_die "FAIL: builder_is_child_build should be false but was $_builder_is_child for the parent script" else - builder_echo "PASS: builder_is_child_build is false ($_builder_is_child) for the parent script" + builder_echo green " ✓ PASS: builder_is_child_build is false ($_builder_is_child) for the parent script" fi # All child actions will generate files which we need to verify for test @@ -41,7 +41,7 @@ function test_present() { if [ ! -f $target.$action ]; then builder_die "$CROSS FAIL: ./$target.$action to be present" else - echo -e "$CHECK PASS: ./$target.$action found as expected" + builder_echo green " ✓ PASS: ./$target.$action found as expected" fi fi } @@ -80,4 +80,4 @@ test_present child1 install builder_run_child_actions error -echo Done \ No newline at end of file +echo Done diff --git a/resources/build/test/trees/child1/build.sh b/resources/build/test/trees/child1/build.sh index ef30f8a517..82cfc1d03d 100755 --- a/resources/build/test/trees/child1/build.sh +++ b/resources/build/test/trees/child1/build.sh @@ -24,7 +24,7 @@ builder_parse "$@" if ! builder_is_child_build; then builder_die "FAIL: builder_is_child_build should return true but was $_builder_is_child for a child script" else - builder_echo "PASS: builder_is_child_build is true ($_builder_is_child) for the child script" + builder_echo green " ✓ PASS: builder_is_child_build is true ($_builder_is_child) for the child script" fi function test_action() { @@ -44,4 +44,4 @@ test_action install if builder_start_action error; then builder_die "This error action is supposed to die" -fi \ No newline at end of file +fi diff --git a/resources/build/test/trees/dep/build.sh b/resources/build/test/trees/dep/build.sh index 2dd8f8ede3..9ef0e9fe4e 100755 --- a/resources/build/test/trees/dep/build.sh +++ b/resources/build/test/trees/dep/build.sh @@ -20,7 +20,7 @@ builder_parse "$@" if builder_is_child_build; then builder_die "FAIL: builder_is_child_build should return false but was $_builder_is_child for a dependency script" else - builder_echo "PASS: builder_is_child_build is false ($_builder_is_child) for the dependency script" + builder_echo green " ✓ PASS: builder_is_child_build is false ($_builder_is_child) for the dependency script" fi function test_action() { diff --git a/resources/build/test/zip.inc.tests.sh b/resources/build/test/zip.inc.tests.sh new file mode 100755 index 0000000000..5b0caa3cf2 --- /dev/null +++ b/resources/build/test/zip.inc.tests.sh @@ -0,0 +1,259 @@ +#!/usr/bin/env bash +# Unit tests for zip.inc.sh + +## START STANDARD BUILD SCRIPT INCLUDE +# adjust relative paths as necessary +THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" +. "${THIS_SCRIPT%/*}/../../../resources/build/build-utils.sh" +# END STANDARD BUILD SCRIPT INCLUDE + +# shellcheck disable=2154 +. "${KEYMAN_ROOT}/resources/build/zip.inc.sh" +. "${KEYMAN_ROOT}/resources/build/test/test-utils.inc.sh" + +# Mock command -v to simulate presence/absence of zip +function command() { + if [[ "$1" == "-v" && "$2" == "zip" ]]; then + if [[ "${MOCK_ZIP_PRESENT:-1}" == "1" ]]; then + echo "/usr/bin/zip" + return 0 + else + return 1 + fi + fi + builtin command "$@" +} + +function setup() { + # Mock zip and 7z commands to capture invocation + ZIP_CMD_LOG="" + function zip() { + ZIP_CMD_LOG="zip $*" + return 0 + } + function 7z() { + ZIP_CMD_LOG="7z $*" + return 0 + } + # Mock /custom/sevenz/7z.exe for Windows test + function /custom/sevenz/7z.exe() { + ZIP_CMD_LOG="/custom/sevenz/7z.exe $*" + return 0 + } +} + +function teardown() { + unset zip + unset 7z + unset /custom/sevenz/7z.exe + reset_zip_test_env +} + +setup + +# Helper to reset log and env +function reset_zip_test_env() { + ZIP_CMD_LOG="" + unset SEVENZ + unset SEVENZ_HOME + MOCK_ZIP_PRESENT=1 + export OSTYPE="linux" +} + +function test_add_zip_files_with_zip_basic() { + # Setup + reset_zip_test_env + MOCK_ZIP_PRESENT=1 + + # Execute + add_zip_files "archive.zip" file1.txt file2.txt + + # Verify + assert-equal "${ZIP_CMD_LOG}" "zip archive.zip file1.txt file2.txt" "add_zip_files zip basic invocation" +} + +function test_add_zip_files_with_zip_flags() { + # Setup + reset_zip_test_env + MOCK_ZIP_PRESENT=1 + + # Execute + add_zip_files "archive.zip" -r -q -9 file1.txt + + # Verify + # -q and -9 are passed to zip, not 7z + assert-equal "${ZIP_CMD_LOG}" "zip -r -q -9 archive.zip file1.txt" "add_zip_files zip with flags" +} + +function test_add_zip_files_with_zip_exclude_flag() { + # Setup + reset_zip_test_env + MOCK_ZIP_PRESENT=1 + + # Execute + add_zip_files "archive.zip" -x@exclude.lst file1.txt + + # Verify + assert-equal "${ZIP_CMD_LOG}" "zip -x@exclude.lst archive.zip file1.txt" "add_zip_files zip with exclude flag" +} + +function test_add_zip_files_with_7z_basic() { + # Setup + reset_zip_test_env + MOCK_ZIP_PRESENT=0 + + # Execute + add_zip_files "archive.7z" file1.txt file2.txt + + # Verify + assert-equal "${ZIP_CMD_LOG}" "7z a archive.7z file1.txt file2.txt" "add_zip_files 7z basic invocation" +} + +function test_add_zip_files_with_7z_flags() { + # Setup + reset_zip_test_env + MOCK_ZIP_PRESENT=0 + + # Execute + add_zip_files "archive.7z" -r -q -5 file1.txt + + # Verify + # -r is passed, -q becomes -bd -bb0, -5 becomes -mx5 + assert-equal "${ZIP_CMD_LOG}" "7z a -r -bd -bb0 -mx5 archive.7z file1.txt" "add_zip_files 7z with flags" +} + +function test_add_zip_files_with_7z_exclude_flag() { + # Setup + reset_zip_test_env + MOCK_ZIP_PRESENT=0 + + # Execute + add_zip_files "archive.7z" -x@exclude.lst file1.txt + + # Verify + assert-equal "${ZIP_CMD_LOG}" "7z a -x@exclude.lst archive.7z file1.txt" "add_zip_files 7z with exclude flag" +} + +function test_add_zip_files_with_7z_on_windows() { + # Setup + reset_zip_test_env + MOCK_ZIP_PRESENT=0 + export OSTYPE="msys" + export SEVENZ_HOME="/custom/sevenz" + unset SEVENZ + + # Execute + add_zip_files "archive.7z" file1.txt + + # Verify + assert-equal "${ZIP_CMD_LOG}" "/custom/sevenz/7z.exe a archive.7z file1.txt" "add_zip_files 7z on windows" +} + +function _create_files() { + local FILEDIR="$1" + mkdir -p "${FILEDIR}/a/b/c" + touch "${FILEDIR}/file1.sh" + touch "${FILEDIR}/file2.sh" + touch "${FILEDIR}/build.sh" + touch "${FILEDIR}/a/file3.sh" + touch "${FILEDIR}/a/build.sh" + touch "${FILEDIR}/a/b/file4.sh" + touch "${FILEDIR}/a/b/build.sh" + touch "${FILEDIR}/a/b/c/file5.sh" + + EXPECTED_CONTENT=". +./a +./a/b +./a/b/c +./a/b/c/file5.sh +./a/b/file4.sh +./a/file3.sh +./file1.sh +./file2.sh" +} + +function test_add_zip_files_with_zip_real() { + if ! builtin command -v zip >/dev/null 2>&1; then + builder_echo warning " IGNORE: add_zip_files zip with real data" + return 0 + fi + + # Setup + teardown + MOCK_ZIP_PRESENT=1 + + ARCHIVE=$(mktemp -u --suffix=.zip) + FILEDIR=$(mktemp -d) + _create_files "${FILEDIR}" + + # Execute + ( + # shellcheck disable=2164 + cd "${FILEDIR}" + add_zip_files "${ARCHIVE}" -r -q -xr!build.sh file1.sh file2.sh a/ # > /dev/null + ) + + # Verify + NEWDIR=$(mktemp -d) + ( + # shellcheck disable=2164 + cd "${NEWDIR}" + unzip "${ARCHIVE}" > /dev/null + ARCHIVE_CONTENT=$(find . | sort) + assert-equal "${ARCHIVE_CONTENT}" "${EXPECTED_CONTENT}" "add_zip_files zip with real data" --quiet + ) + + rm -rf "${FILEDIR}" + rm -rf "${NEWDIR}" + rm "${ARCHIVE}" + setup +} + +function test_add_zip_files_with_7z_real() { + if ! builtin command -v 7z >/dev/null 2>&1; then + builder_echo warning " IGNORE: add_zip_files 7z with real data" + return 0 + fi + + # Setup + teardown + MOCK_ZIP_PRESENT=0 + + ARCHIVE=$(mktemp -u --suffix=.zip) + FILEDIR=$(mktemp -d) + _create_files "${FILEDIR}" + + # Execute + ( + # shellcheck disable=2164 + cd "${FILEDIR}" + add_zip_files "${ARCHIVE}" -r -q -xr!build.sh file1.sh file2.sh a/ > /dev/null + # 7z a -bd -bb0 "${ARCHIVE}" file1.sh file2.sh a/ -xr!build.sh + ) + + # Verify + NEWDIR=$(mktemp -d) + ( + # shellcheck disable=2164 + cd "${NEWDIR}" + 7z x "${ARCHIVE}" > /dev/null + ARCHIVE_CONTENT=$(find . | sort) + assert-equal "${ARCHIVE_CONTENT}" "${EXPECTED_CONTENT}" "add_zip_files 7z with real data" --quiet + ) + + rm -rf "${FILEDIR}" + rm -rf "${NEWDIR}" + rm "${ARCHIVE}" + setup +} + +# Run all tests +test_add_zip_files_with_zip_basic +test_add_zip_files_with_zip_flags +test_add_zip_files_with_zip_exclude_flag +test_add_zip_files_with_zip_real +test_add_zip_files_with_7z_basic +test_add_zip_files_with_7z_flags +test_add_zip_files_with_7z_exclude_flag +test_add_zip_files_with_7z_on_windows +test_add_zip_files_with_7z_real diff --git a/resources/build/zip.inc.sh b/resources/build/zip.inc.sh index 1f80a71edb..a2c5736aee 100644 --- a/resources/build/zip.inc.sh +++ b/resources/build/zip.inc.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +# shellscript shell=bash # # This script contains zip and unzip (TODO) utilities to function for all environments. # unzip is available in all environments @@ -12,8 +12,10 @@ # [zip filename] # [list of flags to pass to zip command] Flags start with a single-dash # -x@filename for a file containing list of files to exclude from the archive +# -xr!name to exclude files matching name from the archive # -* all other flags -# Flags passed in are treated as zip parameters, and internally converterted to 7z flags as applicable +# Flags passed in are treated as zip parameters, and internally converterted +# to 7z flags as applicable # [list of files to include in zip] function add_zip_files() { @@ -27,6 +29,7 @@ function add_zip_files() { local ZIP_FLAGS=() local SEVENZ_FLAGS=('a') # 7z requires a command local INCLUDE=() + local EXCLUDE_FILE while [[ $# -gt 0 ]] ; do case "$1" in -r) @@ -41,6 +44,14 @@ function add_zip_files() { SEVENZ_FLAGS+=($1) shift ;; + -xr!*) + # Recursively exclude the file after -xr! from the archive + EXCLUDE_FILE=$(mktemp) + find . -name ${1##-xr!} > "${EXCLUDE_FILE}" + ZIP_FLAGS+=("-x@${EXCLUDE_FILE}") + SEVENZ_FLAGS+=($1) + shift + ;; # Zip flags that have a corresponding 7z flag -q) @@ -51,14 +62,14 @@ function add_zip_files() { shift ;; -[0123456789]) - # Compression level where + # Compression level where # -0 indicates no compression # -1 indicates low compression (fastest) # -9 indicates ultra compression (slowest) ZIP_FLAGS+=($1) if [[ $1 =~ -([0-9]) ]]; then SEVENZ_FLAGS+=("-mx${BASH_REMATCH[1]}") - fi + fi shift; ;; @@ -102,4 +113,7 @@ function add_zip_files() { # builder_echo_debug "${COMPRESS_CMD} ${SEVENZ_FLAGS[@]} ${ZIP_FLAGS[@]} ${ZIP_FILE} ${INCLUDE[@]}" "${COMPRESS_CMD}" ${SEVENZ_FLAGS[@]} ${ZIP_FLAGS[@]} ${ZIP_FILE} ${INCLUDE[@]} + if [[ -n "${EXCLUDE_FILE:-}" ]]; then + rm "${EXCLUDE_FILE}" + fi } diff --git a/resources/teamcity/android/keyman-android-release.sh b/resources/teamcity/android/keyman-android-release.sh index 856a86424e..b12356606c 100755 --- a/resources/teamcity/android/keyman-android-release.sh +++ b/resources/teamcity/android/keyman-android-release.sh @@ -14,6 +14,7 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" # shellcheck disable=SC2154 . "${KEYMAN_ROOT}/resources/shellHelperFunctions.sh" +. "${KEYMAN_ROOT}/resources/zip.inc.sh" . "${KEYMAN_ROOT}/resources/teamcity/includes/tc-helpers.inc.sh" . "${KEYMAN_ROOT}/resources/teamcity/android/android-actions.inc.sh" @@ -45,7 +46,6 @@ function _publish_to_downloads_keyman_com() { builder_echo start "publish to downloads.keyman.com" "Publishing release to downloads.keyman.com" local UPLOAD_PATH KEYMAN_ENGINE_ANDROID_ZIP KEYMAN_APK FIRSTVOICES_APK - local COMPRESS_CMD # shellcheck disable=SC2154 UPLOAD_PATH="${KEYMAN_ROOT}/android/upload/${KEYMAN_VERSION}" @@ -53,15 +53,12 @@ function _publish_to_downloads_keyman_com() { KEYMAN_APK="keyman-${KEYMAN_VERSION}.apk" FIRSTVOICES_APK="firstvoices-${KEYMAN_VERSION}.apk" - # shellcheck disable=SC2154 - COMPRESS_CMD="${SEVENZ_HOME}/7z" - rm -rf "${UPLOAD_PATH}" mkdir -p "${UPLOAD_PATH}" ( cd "${KEYMAN_ROOT}/android/KMAPro/kMAPro/libs" - "${COMPRESS_CMD}" a -bd -bb0 "../../../${UPLOAD_PATH}/${KEYMAN_ENGINE_ANDROID_ZIP}" keyman-engine.aar ../../../Samples '-xr!build.sh' + add_zip_files -q -xr!build.sh "${UPLOAD_PATH}/${KEYMAN_ENGINE_ANDROID_ZIP}" keyman-engine.aar "${KEYMAN_ROOT}/android/Samples" ) cp "${KEYMAN_ROOT}/android/KMAPro/kMAPro/build/outputs/apk/release/${KEYMAN_APK}" "${UPLOAD_PATH}" diff --git a/resources/teamcity/developer/keyman-developer-release.sh b/resources/teamcity/developer/keyman-developer-release.sh index 35f27ec6e4..119ba283aa 100755 --- a/resources/teamcity/developer/keyman-developer-release.sh +++ b/resources/teamcity/developer/keyman-developer-release.sh @@ -93,9 +93,9 @@ function _publish_to_downloads_keyman_com() { ( cd "${KEYMAN_ROOT}/developer" - local UPLOAD_PATH KEYBOARDS_PATH KMCOMP_ZIP DEVELOPER_EXE DEBUG_ZIP COMPRESS_CMD + local UPLOAD_PATH KEYBOARDS_PATH KMCOMP_ZIP DEVELOPER_EXE DEBUG_ZIP # shellcheck disable=SC2154 - UPLOAD_PATH="upload/${KEYMAN_VERSION}" + UPLOAD_PATH="${KEYMAN_ROOT}/developer/upload/${KEYMAN_VERSION}" KEYBOARDS_PATH="${UPLOAD_PATH}/keyboards" KMCOMP_ZIP="kmcomp-${KEYMAN_VERSION}.zip" DEVELOPER_EXE="keymandeveloper-${KEYMAN_VERSION}.exe" @@ -104,17 +104,8 @@ function _publish_to_downloads_keyman_com() { rm -rf "${UPLOAD_PATH}" mkdir -p "${UPLOAD_PATH}" - ( - cd bin - cp "${KEYMAN_ROOT}/common/schemas/keyboard_info/keyboard_info.schema.json" . - - # shellcheck disable=SC2154 - COMPRESS_CMD="${SEVENZ_HOME}/7z" - - "${COMPRESS_CMD}" a -bd -bb0 "../${UPLOAD_PATH}/${KMCOMP_ZIP}" kmconvert.exe keyboard_info.schema.json xml/layoutbuilder/*.keyman-touch-layout projects/ server/ - ) - - cp "release/${KEYMAN_VERSION}/${DEVELOPER_EXE}" "${UPLOAD_PATH}/" + cp "${KEYMAN_ROOT}/developer/release/${KEYMAN_VERSION}/${DEVELOPER_EXE}" "${UPLOAD_PATH}/" + cp "${KEYMAN_ROOT}/developer/release/${KEYMAN_VERSION}/${KMCOMP_ZIP}" "${UPLOAD_PATH}/" write_download_info "${UPLOAD_PATH}" "${DEVELOPER_EXE}" "Keyman Developer" exe win write_download_info "${UPLOAD_PATH}" "${KMCOMP_ZIP}" "Keyman Developer Command-Line Compiler" zip win @@ -122,12 +113,12 @@ function _publish_to_downloads_keyman_com() { mkdir -p "${KEYBOARDS_PATH}" cp -r "${KEYMAN_ROOT}/common/test/keyboards"/*/build/*.kmp "${KEYBOARDS_PATH}/" - if [[ -f "release/${KEYMAN_VERSION}/${DEBUG_ZIP}" ]]; then - cp "release/${KEYMAN_VERSION}/${DEBUG_ZIP}" "${UPLOAD_PATH}/" + if [[ -f "${KEYMAN_ROOT}/developer/release/${KEYMAN_VERSION}/${DEBUG_ZIP}" ]]; then + cp "${KEYMAN_ROOT}/developer/release/${KEYMAN_VERSION}/${DEBUG_ZIP}" "${UPLOAD_PATH}/" write_download_info "${UPLOAD_PATH}" "${DEBUG_ZIP}" "Keyman Developer debug files" zip win fi - cd upload + cd "${KEYMAN_ROOT}/developer/upload" # shellcheck disable=SC2154 tc_rsync_upload "${KEYMAN_VERSION}" "developer/${KEYMAN_TIER}" ) diff --git a/resources/teamcity/includes/tc-mac.inc.sh b/resources/teamcity/includes/tc-mac.inc.sh index 69d805bb0a..b5d2f23368 100644 --- a/resources/teamcity/includes/tc-mac.inc.sh +++ b/resources/teamcity/includes/tc-mac.inc.sh @@ -18,7 +18,7 @@ ba_mac_clean_xcode_derived_data() { ba_mac_unmount_volumes_keyman() { builder_echo start "unmount" "Unmounting Keyman volumes" if [[ -d /Volumes/Keyman ]]; then - hdiutil detach /Volumes/Keyman; + hdiutil detach /Volumes/Keyman fi - builder_echo end success "unmount" "Finished unmounting Keyman volumes" + builder_echo end "unmount" success "Finished unmounting Keyman volumes" } diff --git a/resources/teamcity/macos/keyman-macos-release.sh b/resources/teamcity/macos/keyman-macos-release.sh index b20005bdfb..d9e9e09c19 100755 --- a/resources/teamcity/macos/keyman-macos-release.sh +++ b/resources/teamcity/macos/keyman-macos-release.sh @@ -62,15 +62,7 @@ function _publish_to_downloads_keyman_com() { builder_echo end "publish to downloads.keyman.com" success "Finished publishing release to downloads.keyman.com" } -function _build_publish() { - builder_echo start "publish" "Publishing Keyman for macOS" - # shellcheck disable=SC2154 - "${KEYMAN_ROOT}/mac/build.sh" publish - builder_echo end success "publish" "Finished publishing Keyman for macOS" -} - function do_publish() { - _build_publish _publish_to_downloads_keyman_com upload_help "Keyman for macOS" mac } diff --git a/resources/teamcity/macos/macos-actions.inc.sh b/resources/teamcity/macos/macos-actions.inc.sh index 84095beac7..832de1aebe 100755 --- a/resources/teamcity/macos/macos-actions.inc.sh +++ b/resources/teamcity/macos/macos-actions.inc.sh @@ -10,5 +10,5 @@ macos_build_action() { builder_echo start "build" "Building Keyman for macOS" # shellcheck disable=SC2154 "${KEYMAN_ROOT}/mac/build.sh" configure build test publish - builder_echo end success "build" "Finished building Keyman for macOS" + builder_echo end "build" success "Finished building Keyman for macOS" } diff --git a/web/ci.sh b/web/ci.sh index f6a92b943b..77210ae224 100755 --- a/web/ci.sh +++ b/web/ci.sh @@ -13,6 +13,7 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" ## END STANDARD BUILD SCRIPT INCLUDE . "${KEYMAN_ROOT}/resources/shellHelperFunctions.sh" +. "${KEYMAN_ROOT}/resources/build/zip.inc.sh" . "${KEYMAN_ROOT}/resources/build/ci/pull-requests.inc.sh" # This script runs from its own folder @@ -154,32 +155,12 @@ function prepare_downloads_keyman_com_action() { mkdir -p "${UPLOAD_PATH}" - # On Windows, we use 7-zip (SEVENZ_HOME env var). On other platforms, we use zip. - - COMPRESS_CMD= - COMPRESS_ADD= - - # Marc's preference; use $SEVENZ_HOME and have the BAs set up with THAT as an env var. - if [[ ! -z "${SEVENZ_HOME+x}" ]]; then - COMPRESS_CMD="${SEVENZ_HOME}/7z" - COMPRESS_ADD="a -bd -bb0 -r" # add, hide progress, log level 0, recursive - fi - - if [[ -z "${COMPRESS_CMD}" ]] ; then - if command -v zip &> /dev/null; then - # Note: does not support within-archive renames! - COMPRESS_CMD=zip - COMPRESS_ADD="-r" - else - builder_die "7z and zip commands are both unavailable" - fi - fi - - pushd build/publish - # Zip both the 'debug' and 'release' configurations together. - # shellcheck disable=SC2086 - "${COMPRESS_CMD}" ${COMPRESS_ADD} "${ZIP}" ./* - popd + ( + # shellcheck disable=2164 + cd build/publish + # Zip both the 'debug' and 'release' configurations together. + add_zip_files -q -r "${ZIP}" ./* + ) # --- Second action artifact - the 'static' folder (hosted user testing on downloads.keyman.com) ---