From f339ceb72235a089d64ae9e2ce6ad57b00972ffb Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Tue, 26 May 2026 15:44:44 +0200 Subject: [PATCH] maint(linux): fix Linux source tarball Previously we tried to include all necessary files in the Linux source tarball, so that a user would be able to successfully run the top-level `build.sh` file. However, that causes problems when new dependencies between sub-projects get introduced that would require adding additional sub-projects to the tarball. This change modifies the tarball to only include files that are necessary to run the `linux/build.sh` file, i.e. necessary to build Keyman for Linux. This is likely what users would expect when they download a source tarball for Linux. If users want to build other parts of Keyman that are buildable on Linux they can download the source tarball that the GitHub releases page provides, or clone the repo. In order to be able to still run the top-level `build.sh` script, this change replaces the top-level `build.sh` with a script that simply forwards the arguments to `linux/build.sh`. This change also reverts the recent introduction of a *.pkg.tar.xz file that contained all files necessary to build a Debian/Ubuntu package. There wouldn't be a difference anymore between the two .tar.xz files. Fixes: #16004 Fixes: #16005 Build-bot: skip build:linux Test-bot: skip --- linux/scripts/dist.sh | 224 ++++++++---------- linux/scripts/package-build.inc.sh | 2 +- linux/scripts/verify_source.sh | 18 +- linux/scripts/watch.in | 2 +- .../teamcity/linux/keyman-linux-release.sh | 7 +- 5 files changed, 108 insertions(+), 145 deletions(-) diff --git a/linux/scripts/dist.sh b/linux/scripts/dist.sh index b66615ad9d..a09225e3ea 100755 --- a/linux/scripts/dist.sh +++ b/linux/scripts/dist.sh @@ -15,6 +15,101 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" # shellcheck disable=SC2154 . "${KEYMAN_ROOT}/linux/scripts/package-build.inc.sh" +create_tarball() { + # Include these files and folders: + # shellcheck disable=2034 # to_include appears to be unused, even though + # it is used indirectly by generate_tar_ignore_list. + to_include=( + common/build.sh \ + common/cpp \ + common/include \ + common/linux \ + common/test/keyboards/baseline \ + core \ + linux \ + resources/build/*.sh \ + resources/build/meson \ + resources/standards-data \ + resources/*.sh \ + ./*.md \ + ./build.sh \ + ./*.json \ + ) + + # files and subfolders to exclude from paths included in 'to_include', + # i.e. the exceptions to 'to_include'. + # shellcheck disable=2034 # to_exclude appears to be unused, even though + # it is used indirectly by generate_tar_ignore_list. + to_exclude=( + build \ + common/test/keyboards/baseline/kmcomp-*.zip \ + linux/builddebs \ + linux/docs/help \ + linux/keyman-config/keyman_config/version.py \ + linux/keyman-config/buildtools/build-langtags.py \ + linux/upload \ + ) + + # array to store list of --tar-ignore parameters generated from to_include and to_exclude. + ignored_files=() + + generate_tar_ignore_list "./" to_include to_exclude ignored_files "$(basename "${KEYMAN_ROOT}")" + + # Note: explicitly specify the --tar-ignores here for files/folders that we always + # want to ignore regardless of their location. Having them here allows us to pass + # the wildcards to dpkg-source - whereas the wildcards in 'to_exclude' will be + # resolved and replaced with multiple --tar-ignore entries. + dpkg-source \ + --tar-ignore=*~ \ + --tar-ignore=.git \ + --tar-ignore=.gitattributes \ + --tar-ignore=.gitignore \ + --tar-ignore=experiments \ + --tar-ignore=debian \ + --tar-ignore=.github \ + --tar-ignore=.vscode \ + --tar-ignore=.configured \ + --tar-ignore=.devcontainer \ + --tar-ignore=.pc \ + --tar-ignore=__pycache__ \ + --tar-ignore=node_modules \ + --tar-ignore=keyman_1* \ + --tar-ignore=launchpad \ + --tar-ignore=dist \ + --tar-ignore=VERSION \ + \ + "${ignored_files[@]}" \ + \ + --compression=xz --build . + mv ../keyman_"${KEYMAN_VERSION}".tar.xz linux/dist/keyman-"${KEYMAN_VERSION}".tar.xz +} + +replace_toplevel_buildsh() { + # extract the tarball and replace top-level build.sh, then recreate tarball + builder_echo heading "Replacing top-level build.sh" + cd "${KEYMAN_ROOT}/linux/dist" + tar xfJ keyman-"${KEYMAN_VERSION}".tar.xz + cat > "keyman/build.sh" << EOF +#!/usr/bin/env bash +linux/build.sh "\$@" +EOF + chmod +x "keyman/build.sh" + tar cfJ "keyman-${KEYMAN_VERSION}.tar.xz" "keyman" + rm -rf "keyman" +} + +create_debian_origtarxz() { + builder_echo heading "Creating Debian orig.tar.xz" + cd "${KEYMAN_ROOT}/linux/dist" + pkgvers="keyman-${KEYMAN_VERSION}" + tar xfJ keyman-"${KEYMAN_VERSION}".tar.xz + mv -v keyman "${pkgvers}" 2>/dev/null || mv -v "$(find . -mindepth 1 -maxdepth 1 -type d)" "${pkgvers}" + tar cfJ "keyman_${KEYMAN_VERSION}.orig.tar.xz" "${pkgvers}" + rm "keyman-${KEYMAN_VERSION}.tar.xz" + rm -rf "${pkgvers}" +} + + BASEDIR=$(pwd) cd "${KEYMAN_ROOT}/linux" @@ -24,6 +119,8 @@ if [[ ! -z ${1+x} ]] && [[ "$1" == "origdist" ]]; then shift fi +builder_echo heading "Creating source tarball for Keyman ${KEYMAN_VERSION}" + rm -rf dist mkdir -p dist @@ -34,130 +131,13 @@ echo "3.0 (native)" > debian/source/format # shellcheck disable=SC2154 dch keyman --newversion "${KEYMAN_VERSION}" --force-bad-version --nomultimaint -# Create the tarball - -# We always include these files which are the minimum files and -# folder required for Ubuntu/Debian packaging -# shellcheck disable=2034 # to_include appears to be unused -to_include=( - common/build.sh \ - common/cpp \ - common/include \ - common/linux \ - common/test/keyboards/baseline \ - core \ - linux \ - resources/build/*.sh \ - resources/build/meson \ - resources/standards-data \ - resources/*.sh \ - ./*.md \ - ./build.sh \ - ./*.json \ -) - -# files and subfolders to exclude from paths included in 'to_include', -# i.e. the exceptions to 'to_include'. - -# shellcheck disable=2034 # to_exclude appears to be unused -to_exclude=( - build \ - common/test/keyboards/baseline/kmcomp-*.zip \ - linux/builddebs \ - linux/docs/help \ - linux/keyman-config/keyman_config/version.py \ - linux/keyman-config/buildtools/build-langtags.py \ - linux/upload \ -) - -if [[ -z "${create_origdist+x}" ]]; then - # If we build a full source tarball we include additional files - # so that it's possible to run `${KEYMAN_ROOT}/build.sh` on Linux - - # shellcheck disable=2034 # to_include appears to be unused - to_include+=( - common/schemas \ - common/tools/hextobin \ - common/web/keyman-version \ - common/web/langtags \ - common/web/types \ - common/windows/cpp \ - common/windows/include \ - developer/src/common/include \ - developer/src/common/web \ - developer/src/ext/json \ - developer/src/kmc \ - developer/src/kmc-analyze \ - developer/src/kmc-copy \ - developer/src/kmc-generate \ - developer/src/kmc-keyboard-info \ - developer/src/kmc-kmn \ - developer/src/kmc-ldml \ - developer/src/kmc-model \ - developer/src/kmc-model-info \ - developer/src/kmc-package \ - developer/src/kmcmplib \ - docs/minimum-versions.md.in - resources/build \ - resources/standards-data \ - ) - - # additional files and subfolders to exclude from paths included in 'to_include', - # i.e. the exceptions to 'to_include'. - # shellcheck disable=2034 # to_exclude appears to be unused - to_exclude+=( - *.exe \ - resources/build/history \ - resources/build/l10n \ - resources/build/mac \ - resources/build/win \ - resources/build/*.lua \ - ) -fi - -# array to store list of --tar-ignore parameters generated from to_include and to_exclude. -ignored_files=() - -generate_tar_ignore_list "./" to_include to_exclude ignored_files "$(basename "${KEYMAN_ROOT}")" - -# Note: explicitly specify the --tar-ignores here for files/folders that we always -# want to ignore regardless of their location. Having them here allows us to pass -# the wildcards to dpkg-source - whereas the wildcards in 'to_exclude' will be -# resolved and replaced with multiple --tar-ignore entries. -dpkg-source \ - --tar-ignore=*~ \ - --tar-ignore=.git \ - --tar-ignore=.gitattributes \ - --tar-ignore=.gitignore \ - --tar-ignore=experiments \ - --tar-ignore=debian \ - --tar-ignore=.github \ - --tar-ignore=.vscode \ - --tar-ignore=.configured \ - --tar-ignore=.devcontainer \ - --tar-ignore=.pc \ - --tar-ignore=__pycache__ \ - --tar-ignore=node_modules \ - --tar-ignore=keyman_1* \ - --tar-ignore=launchpad \ - --tar-ignore=dist \ - --tar-ignore=VERSION \ - \ - "${ignored_files[@]}" \ - \ - -Zxz -b . - -mv ../keyman_"${KEYMAN_VERSION}".tar.xz linux/dist/keyman-"${KEYMAN_VERSION}".tar.xz +create_tarball echo "3.0 (quilt)" > debian/source/format -cd "${BASEDIR}" +replace_toplevel_buildsh # create orig.tar.xz if [[ ! -z "${create_origdist+x}" ]]; then - cd "${KEYMAN_ROOT}/linux/dist" - pkgvers="keyman-${KEYMAN_VERSION}" - tar xfJ keyman-"${KEYMAN_VERSION}".tar.xz - mv -v keyman "${pkgvers}" 2>/dev/null || mv -v "$(find . -mindepth 1 -maxdepth 1 -type d)" "${pkgvers}" - tar cfJ "keyman_${KEYMAN_VERSION}.orig.tar.xz" "${pkgvers}" - rm "keyman-${KEYMAN_VERSION}.tar.xz" - rm -rf "${pkgvers}" + create_debian_origtarxz fi + +cd "${BASEDIR}" diff --git a/linux/scripts/package-build.inc.sh b/linux/scripts/package-build.inc.sh index ca58fb9d54..f1d6f15c69 100644 --- a/linux/scripts/package-build.inc.sh +++ b/linux/scripts/package-build.inc.sh @@ -32,7 +32,7 @@ function downloadSource() { cd .. mv "keyman-${version}" "${KEYMAN_ROOT}/linux/${packageDir}" mv "keyman_${version}.orig.tar.xz" "${KEYMAN_ROOT}/linux/${packageDir}" - mv "keyman_${version}.pkg.tar.xz" "${KEYMAN_ROOT}/linux/${packageDir}" + mv "keyman-${version}.tar.xz" "${KEYMAN_ROOT}/linux/${packageDir}" mv "keyman"*.asc "${KEYMAN_ROOT}/linux/${packageDir}" rm "keyman"*.debian.tar.xz cd "${KEYMAN_ROOT}/linux/${packageDir}" || exit diff --git a/linux/scripts/verify_source.sh b/linux/scripts/verify_source.sh index bec62599f5..2ed00b9a6f 100755 --- a/linux/scripts/verify_source.sh +++ b/linux/scripts/verify_source.sh @@ -36,10 +36,6 @@ create_source_tarball() { ./scripts/reconf.sh PKG_CONFIG_PATH="${KEYMAN_ROOT}/core/build/arch/release/meson-private" ./scripts/dist.sh mv "dist/keyman-${KEYMAN_VERSION}.tar.xz" "${target_dir}" - - builder_echo heading "Make source for packaging" - PKG_CONFIG_PATH="${KEYMAN_ROOT}/core/build/arch/release/meson-private" ./scripts/dist.sh origdist - mv "dist/keyman_${KEYMAN_VERSION}.orig.tar.xz" "${target_dir}/keyman_${KEYMAN_VERSION}.pkg.tar.xz" } extract_source_tarball() { @@ -51,15 +47,6 @@ extract_source_tarball() { tar -xvf "keyman-${KEYMAN_VERSION}.tar.xz" } -extract_packaging_source_tarball() { - local target_dir="$1" - - builder_echo heading "Extract packaging source tarball" - cd "${target_dir}" - rm -rf "keyman-${KEYMAN_VERSION}" - tar -xvf "keyman_${KEYMAN_VERSION}.pkg.tar.xz" -} - verify_can_build() { local target_dir="$1" builder_echo heading "Verifying build of tarball" @@ -81,7 +68,7 @@ create_source_package() { cd launchpad cp -r "../keyman-${KEYMAN_VERSION}" . cp -r "${KEYMAN_ROOT}/linux/debian" "keyman-${KEYMAN_VERSION}" - cp "${target_dir}/keyman_${KEYMAN_VERSION}.pkg.tar.xz" "keyman_${KEYMAN_VERSION}.orig.tar.xz" + cp "${target_dir}/keyman-${KEYMAN_VERSION}.tar.xz" "keyman_${KEYMAN_VERSION}.orig.tar.xz" "keyman-${KEYMAN_VERSION}/linux/scripts/launchpad.sh" --no-download \ --dist "$(lsb_release -c -s)" --outputdir "${target_dir}/launchpad" --no-lintian --no-sign } @@ -124,7 +111,8 @@ fi if ! builder_has_option --source-only; then builder_echo start lintian "Verifying Launchpad source package" - extract_packaging_source_tarball "${TARGET_DIR}" + extract_source_tarball "${TARGET_DIR}" + mv "${TARGET_DIR}/keyman" "${TARGET_DIR}/keyman-${KEYMAN_VERSION}" create_source_package "${TARGET_DIR}" verify_lintian rm -rf "${TARGET_DIR}/keyman-${KEYMAN_VERSION}" diff --git a/linux/scripts/watch.in b/linux/scripts/watch.in index 82a5ce6cde..b4c0039a84 100644 --- a/linux/scripts/watch.in +++ b/linux/scripts/watch.in @@ -1,3 +1,3 @@ version=4 # Tier replaced by package-build.inc.sh script -opts=pgpsigurlmangle=s/$/.asc/ https://downloads.keyman.com/linux/$tier/@ANY_VERSION@/@PACKAGE@@ANY_VERSION@.pkg@ARCHIVE_EXT@ debian uupdate +opts=pgpsigurlmangle=s/$/.asc/ https://downloads.keyman.com/linux/$tier/@ANY_VERSION@/@PACKAGE@@ANY_VERSION@@ARCHIVE_EXT@ debian uupdate diff --git a/resources/teamcity/linux/keyman-linux-release.sh b/resources/teamcity/linux/keyman-linux-release.sh index 9fb73ef91e..9cc1b76a1b 100755 --- a/resources/teamcity/linux/keyman-linux-release.sh +++ b/resources/teamcity/linux/keyman-linux-release.sh @@ -61,15 +61,11 @@ function _make_release_source_tarball() { ./scripts/reconf.sh PKG_CONFIG_PATH="${KEYMAN_ROOT}/core/build/arch/release/meson-private" ./scripts/dist.sh mv dist/*.tar.xz "upload/${KEYMAN_VERSION}/" - builder_echo heading "Make source for packaging" - PKG_CONFIG_PATH="${KEYMAN_ROOT}/core/build/arch/release/meson-private" ./scripts/dist.sh origdist - mv "dist/keyman_${KEYMAN_VERSION}.orig.tar.xz" "dist/keyman_${KEYMAN_VERSION}.pkg.tar.xz" - mv dist/*.tar.xz "upload/${KEYMAN_VERSION}/" ( cd "upload/${KEYMAN_VERSION}" sha256sum ./*.tar.xz > SHA256SUMS - builder_echo end "make source tarball" success "Make source tarball" ) + builder_echo end "make source tarball" success "Make source tarball" } function _sign_source_tarball() { @@ -100,7 +96,6 @@ function _publish_to_downloads() { chmod a+r "${UPLOAD_DIR}"/* write_download_info "${UPLOAD_DIR}" "keyman-${KEYMAN_VERSION}.tar.xz" "Keyman for Linux source tarball" tar.xz linux - write_download_info "${UPLOAD_DIR}" "keyman_${KEYMAN_VERSION}.pkg.tar.xz" "Keyman for Linux source for packaging" tar.xz linux tc_rsync_upload "${UPLOAD_DIR}" "linux/${KEYMAN_TIER}" builder_echo end "publish to downloads" success "Publish to downloads.keyman.com"