From f3206d58e4ce175ce0b2dc056eff011bee2ccc6a Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Thu, 10 Jul 2025 14:23:57 +0200 Subject: [PATCH 1/3] maint(common): add support for inline file exclusions to `add_zip_files` This change adds support for the `-xr!wildcard` parameter to the `add_zip_files` function, e.g. `-xr!build.sh` will exclude all `build.sh` files from the archive. This is an alternative to adding a file list. --- resources/build/zip.inc.sh | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/resources/build/zip.inc.sh b/resources/build/zip.inc.sh index 1f80a71edb..a84d8e940e 100644 --- a/resources/build/zip.inc.sh +++ b/resources/build/zip.inc.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +# shellscript shell=bash # # This script contains zip and unzip (TODO) utilities to function for all environments. # unzip is available in all environments @@ -27,6 +27,7 @@ function add_zip_files() { local ZIP_FLAGS=() local SEVENZ_FLAGS=('a') # 7z requires a command local INCLUDE=() + local EXCLUDE_FILE while [[ $# -gt 0 ]] ; do case "$1" in -r) @@ -41,6 +42,14 @@ function add_zip_files() { SEVENZ_FLAGS+=($1) shift ;; + -xr!*) + # Recursively exclude the file after -xr! from the archive + EXCLUDE_FILE=$(mktemp) + find . -name ${1##-xr!} > "${EXCLUDE_FILE}" + ZIP_FLAGS+=("-x@${EXCLUDE_FILE}") + SEVENZ_FLAGS+=($1) + shift + ;; # Zip flags that have a corresponding 7z flag -q) @@ -51,14 +60,14 @@ function add_zip_files() { shift ;; -[0123456789]) - # Compression level where + # Compression level where # -0 indicates no compression # -1 indicates low compression (fastest) # -9 indicates ultra compression (slowest) ZIP_FLAGS+=($1) if [[ $1 =~ -([0-9]) ]]; then SEVENZ_FLAGS+=("-mx${BASH_REMATCH[1]}") - fi + fi shift; ;; @@ -102,4 +111,7 @@ function add_zip_files() { # builder_echo_debug "${COMPRESS_CMD} ${SEVENZ_FLAGS[@]} ${ZIP_FLAGS[@]} ${ZIP_FILE} ${INCLUDE[@]}" "${COMPRESS_CMD}" ${SEVENZ_FLAGS[@]} ${ZIP_FLAGS[@]} ${ZIP_FILE} ${INCLUDE[@]} + if [[ -n "${EXCLUDE_FILE:-}" ]]; then + rm "${EXCLUDE_FILE}" + fi } From d863fc4aa7dde057bcc12b5a2ba1210cf1d89ba6 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Thu, 10 Jul 2025 14:24:25 +0200 Subject: [PATCH 2/3] maint(common): add unit tests for `zip.inc.sh` --- resources/build/test/test-utils.inc.sh | 12 +- resources/build/test/zip.inc.tests.sh | 259 +++++++++++++++++++++++++ 2 files changed, 268 insertions(+), 3 deletions(-) create mode 100755 resources/build/test/zip.inc.tests.sh diff --git a/resources/build/test/test-utils.inc.sh b/resources/build/test/test-utils.inc.sh index a73f46773a..48569bedae 100644 --- a/resources/build/test/test-utils.inc.sh +++ b/resources/build/test/test-utils.inc.sh @@ -1,14 +1,20 @@ +# shellcheck shell=bash + function assert-equal() { local actual="$1" local expected="$2" local message= + local actual_to_show="'${actual}' " if [[ $# -gt 2 ]]; then message="$3: " fi + if [[ $# -gt 3 ]]; then + actual_to_show="" + fi - if [[ "$actual" != "$expected" ]]; then - builder_die " ✕ FAIL: ${message}actual result '$actual' should equal expected '$expected'" + if [[ "${actual}" != "${expected}" ]]; then + builder_die " ✕ FAIL: ${message}actual result '${actual}' should equal expected '${expected}'" else - builder_echo green " ✓ PASS: ${message}result '$actual' is correct" + builder_echo green " ✓ PASS: ${message}result ${actual_to_show}is correct" fi } diff --git a/resources/build/test/zip.inc.tests.sh b/resources/build/test/zip.inc.tests.sh new file mode 100755 index 0000000000..5b0caa3cf2 --- /dev/null +++ b/resources/build/test/zip.inc.tests.sh @@ -0,0 +1,259 @@ +#!/usr/bin/env bash +# Unit tests for zip.inc.sh + +## START STANDARD BUILD SCRIPT INCLUDE +# adjust relative paths as necessary +THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" +. "${THIS_SCRIPT%/*}/../../../resources/build/build-utils.sh" +# END STANDARD BUILD SCRIPT INCLUDE + +# shellcheck disable=2154 +. "${KEYMAN_ROOT}/resources/build/zip.inc.sh" +. "${KEYMAN_ROOT}/resources/build/test/test-utils.inc.sh" + +# Mock command -v to simulate presence/absence of zip +function command() { + if [[ "$1" == "-v" && "$2" == "zip" ]]; then + if [[ "${MOCK_ZIP_PRESENT:-1}" == "1" ]]; then + echo "/usr/bin/zip" + return 0 + else + return 1 + fi + fi + builtin command "$@" +} + +function setup() { + # Mock zip and 7z commands to capture invocation + ZIP_CMD_LOG="" + function zip() { + ZIP_CMD_LOG="zip $*" + return 0 + } + function 7z() { + ZIP_CMD_LOG="7z $*" + return 0 + } + # Mock /custom/sevenz/7z.exe for Windows test + function /custom/sevenz/7z.exe() { + ZIP_CMD_LOG="/custom/sevenz/7z.exe $*" + return 0 + } +} + +function teardown() { + unset zip + unset 7z + unset /custom/sevenz/7z.exe + reset_zip_test_env +} + +setup + +# Helper to reset log and env +function reset_zip_test_env() { + ZIP_CMD_LOG="" + unset SEVENZ + unset SEVENZ_HOME + MOCK_ZIP_PRESENT=1 + export OSTYPE="linux" +} + +function test_add_zip_files_with_zip_basic() { + # Setup + reset_zip_test_env + MOCK_ZIP_PRESENT=1 + + # Execute + add_zip_files "archive.zip" file1.txt file2.txt + + # Verify + assert-equal "${ZIP_CMD_LOG}" "zip archive.zip file1.txt file2.txt" "add_zip_files zip basic invocation" +} + +function test_add_zip_files_with_zip_flags() { + # Setup + reset_zip_test_env + MOCK_ZIP_PRESENT=1 + + # Execute + add_zip_files "archive.zip" -r -q -9 file1.txt + + # Verify + # -q and -9 are passed to zip, not 7z + assert-equal "${ZIP_CMD_LOG}" "zip -r -q -9 archive.zip file1.txt" "add_zip_files zip with flags" +} + +function test_add_zip_files_with_zip_exclude_flag() { + # Setup + reset_zip_test_env + MOCK_ZIP_PRESENT=1 + + # Execute + add_zip_files "archive.zip" -x@exclude.lst file1.txt + + # Verify + assert-equal "${ZIP_CMD_LOG}" "zip -x@exclude.lst archive.zip file1.txt" "add_zip_files zip with exclude flag" +} + +function test_add_zip_files_with_7z_basic() { + # Setup + reset_zip_test_env + MOCK_ZIP_PRESENT=0 + + # Execute + add_zip_files "archive.7z" file1.txt file2.txt + + # Verify + assert-equal "${ZIP_CMD_LOG}" "7z a archive.7z file1.txt file2.txt" "add_zip_files 7z basic invocation" +} + +function test_add_zip_files_with_7z_flags() { + # Setup + reset_zip_test_env + MOCK_ZIP_PRESENT=0 + + # Execute + add_zip_files "archive.7z" -r -q -5 file1.txt + + # Verify + # -r is passed, -q becomes -bd -bb0, -5 becomes -mx5 + assert-equal "${ZIP_CMD_LOG}" "7z a -r -bd -bb0 -mx5 archive.7z file1.txt" "add_zip_files 7z with flags" +} + +function test_add_zip_files_with_7z_exclude_flag() { + # Setup + reset_zip_test_env + MOCK_ZIP_PRESENT=0 + + # Execute + add_zip_files "archive.7z" -x@exclude.lst file1.txt + + # Verify + assert-equal "${ZIP_CMD_LOG}" "7z a -x@exclude.lst archive.7z file1.txt" "add_zip_files 7z with exclude flag" +} + +function test_add_zip_files_with_7z_on_windows() { + # Setup + reset_zip_test_env + MOCK_ZIP_PRESENT=0 + export OSTYPE="msys" + export SEVENZ_HOME="/custom/sevenz" + unset SEVENZ + + # Execute + add_zip_files "archive.7z" file1.txt + + # Verify + assert-equal "${ZIP_CMD_LOG}" "/custom/sevenz/7z.exe a archive.7z file1.txt" "add_zip_files 7z on windows" +} + +function _create_files() { + local FILEDIR="$1" + mkdir -p "${FILEDIR}/a/b/c" + touch "${FILEDIR}/file1.sh" + touch "${FILEDIR}/file2.sh" + touch "${FILEDIR}/build.sh" + touch "${FILEDIR}/a/file3.sh" + touch "${FILEDIR}/a/build.sh" + touch "${FILEDIR}/a/b/file4.sh" + touch "${FILEDIR}/a/b/build.sh" + touch "${FILEDIR}/a/b/c/file5.sh" + + EXPECTED_CONTENT=". +./a +./a/b +./a/b/c +./a/b/c/file5.sh +./a/b/file4.sh +./a/file3.sh +./file1.sh +./file2.sh" +} + +function test_add_zip_files_with_zip_real() { + if ! builtin command -v zip >/dev/null 2>&1; then + builder_echo warning " IGNORE: add_zip_files zip with real data" + return 0 + fi + + # Setup + teardown + MOCK_ZIP_PRESENT=1 + + ARCHIVE=$(mktemp -u --suffix=.zip) + FILEDIR=$(mktemp -d) + _create_files "${FILEDIR}" + + # Execute + ( + # shellcheck disable=2164 + cd "${FILEDIR}" + add_zip_files "${ARCHIVE}" -r -q -xr!build.sh file1.sh file2.sh a/ # > /dev/null + ) + + # Verify + NEWDIR=$(mktemp -d) + ( + # shellcheck disable=2164 + cd "${NEWDIR}" + unzip "${ARCHIVE}" > /dev/null + ARCHIVE_CONTENT=$(find . | sort) + assert-equal "${ARCHIVE_CONTENT}" "${EXPECTED_CONTENT}" "add_zip_files zip with real data" --quiet + ) + + rm -rf "${FILEDIR}" + rm -rf "${NEWDIR}" + rm "${ARCHIVE}" + setup +} + +function test_add_zip_files_with_7z_real() { + if ! builtin command -v 7z >/dev/null 2>&1; then + builder_echo warning " IGNORE: add_zip_files 7z with real data" + return 0 + fi + + # Setup + teardown + MOCK_ZIP_PRESENT=0 + + ARCHIVE=$(mktemp -u --suffix=.zip) + FILEDIR=$(mktemp -d) + _create_files "${FILEDIR}" + + # Execute + ( + # shellcheck disable=2164 + cd "${FILEDIR}" + add_zip_files "${ARCHIVE}" -r -q -xr!build.sh file1.sh file2.sh a/ > /dev/null + # 7z a -bd -bb0 "${ARCHIVE}" file1.sh file2.sh a/ -xr!build.sh + ) + + # Verify + NEWDIR=$(mktemp -d) + ( + # shellcheck disable=2164 + cd "${NEWDIR}" + 7z x "${ARCHIVE}" > /dev/null + ARCHIVE_CONTENT=$(find . | sort) + assert-equal "${ARCHIVE_CONTENT}" "${EXPECTED_CONTENT}" "add_zip_files 7z with real data" --quiet + ) + + rm -rf "${FILEDIR}" + rm -rf "${NEWDIR}" + rm "${ARCHIVE}" + setup +} + +# Run all tests +test_add_zip_files_with_zip_basic +test_add_zip_files_with_zip_flags +test_add_zip_files_with_zip_exclude_flag +test_add_zip_files_with_zip_real +test_add_zip_files_with_7z_basic +test_add_zip_files_with_7z_flags +test_add_zip_files_with_7z_exclude_flag +test_add_zip_files_with_7z_on_windows +test_add_zip_files_with_7z_real From 16d38a0e8921646bb1e1fae990f64653f8bff6b6 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Thu, 10 Jul 2025 15:27:30 +0200 Subject: [PATCH 3/3] docs(common): update documentation of `zip.inc.sh` --- resources/build/zip.inc.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/resources/build/zip.inc.sh b/resources/build/zip.inc.sh index a84d8e940e..a2c5736aee 100644 --- a/resources/build/zip.inc.sh +++ b/resources/build/zip.inc.sh @@ -12,8 +12,10 @@ # [zip filename] # [list of flags to pass to zip command] Flags start with a single-dash # -x@filename for a file containing list of files to exclude from the archive +# -xr!name to exclude files matching name from the archive # -* all other flags -# Flags passed in are treated as zip parameters, and internally converterted to 7z flags as applicable +# Flags passed in are treated as zip parameters, and internally converterted +# to 7z flags as applicable # [list of files to include in zip] function add_zip_files() {