diff --git a/.github/workflows/deb-packaging.yml b/.github/workflows/deb-packaging.yml index 6e7fe54cb3..5eb2794bce 100644 --- a/.github/workflows/deb-packaging.yml +++ b/.github/workflows/deb-packaging.yml @@ -29,6 +29,7 @@ env: COLOR_GREEN: "\e[32m" GH_TOKEN: ${{ github.token }} STATUS_CONTEXT: 'Ubuntu Packaging' + STATUS_CONTEXT_SOURCE_VERIFICATION: 'Linux source verification' DEBFULLNAME: 'Keyman GHA packager' DEBEMAIL: 'support@keyman.com' _builder_timings: false @@ -403,3 +404,92 @@ jobs: -f target_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \ -f description="$MSG" \ -f context="$STATUS_CONTEXT" + + #--------------------------------------------------------------------------------------------- + source_verification: + name: Verify source tarball and Linux source package + if: github.repository == 'keymanapp/keyman' || github.event.client_payload.force + needs: sourcepackage + runs-on: ubuntu-latest + outputs: + KEYMAN_VERSION: ${{ needs.sourcepackage.outputs.KEYMAN_VERSION }} + + steps: + - name: Set pending status on PR builds + id: set_status + if: github.event.client_payload.isTestBuild == 'true' + shell: bash + run: | + gh api \ + --method POST \ + -H "Accept: application/vnd.github+json" \ + /repos/$GITHUB_REPOSITORY/statuses/${{ github.event.client_payload.buildSha }} \ + -f state='pending' \ + -f target_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \ + -f description='Linux source verification started' \ + -f context="$STATUS_CONTEXT_SOURCE_VERIFICATION" + + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: '${{ github.event.client_payload.buildSha }}' + fetch-depth: 0 + clean: false + path: ${{ github.workspace }}/keyman/ + + - name: Install devscripts + uses: ./keyman/.github/actions/apt-install + with: + packages: devscripts equivs + + - name: Install dependencies + run: | + cd ${{ github.workspace }}/keyman/linux + ./scripts/deb-packaging.sh --gha dependencies + + - name: Verify source tarball + shell: bash + run: | + cd ${{ github.workspace }}/keyman/linux + ./scripts/verify_source.sh --source-only --target-dir "${{ github.workspace }}/keyman-${{ needs.sourcepackage.outputs.KEYMAN_VERSION }}" + + - name: Verify Ubuntu source package + shell: bash + run: | + cd ${{ github.workspace }}/keyman/linux + ./scripts/verify_source.sh --no-create-tarball --launchpad-only --target-dir "${{ github.workspace }}/keyman-${{ needs.sourcepackage.outputs.KEYMAN_VERSION }}" + + set_status_source_verification: + name: Set result status of source verification on PR builds + needs: [source_verification] + runs-on: ubuntu-latest + if: ${{ always() && github.event.client_payload.isTestBuild == 'true' }} + steps: + - name: Set success + if: needs.source_verification.result == 'success' + run: | + echo "RESULT=success" >> $GITHUB_ENV + echo "MSG=Linux source verification succeeded" >> $GITHUB_ENV + + - name: Set cancelled + if: needs.source_verification.result == 'cancelled' + run: | + echo "RESULT=error" >> $GITHUB_ENV + echo "MSG=Linux source verification cancelled" >> $GITHUB_ENV + + - name: Set failure + if: needs.source_verification.result == 'failure' + run: | + echo "RESULT=failure" >> $GITHUB_ENV + echo "MSG=Linux source verification failed" >> $GITHUB_ENV + + - name: Set final status + run: | + gh api \ + --method POST \ + -H "Accept: application/vnd.github+json" \ + /repos/$GITHUB_REPOSITORY/statuses/${{ github.event.client_payload.buildSha }} \ + -f state="$RESULT" \ + -f target_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \ + -f description="$MSG" \ + -f context="$STATUS_CONTEXT_SOURCE_VERIFICATION" diff --git a/.github/workflows/linux-source-verification.yml b/.github/workflows/linux-source-verification.yml deleted file mode 100644 index d2e8a07801..0000000000 --- a/.github/workflows/linux-source-verification.yml +++ /dev/null @@ -1,128 +0,0 @@ -# Keyman is copyright (C) SIL Global. MIT License. -# -# This workflow verifies two things: -# - the source tarball that gets uploaded to downloads.keyman.com. It -# verifies that it contains all files needed to run `build.sh` -# - the source package that gets uploaded to launchpad. It verifies that -# running lintian doesn't output any errors. - -name: "Linux source verification" -run-name: "Linux source verification - ${{ github.event.client_payload.branch }} (branch ${{ github.event.client_payload.baseBranch }}), by @${{ github.event.client_payload.user }}, testbuild: ${{ github.event.client_payload.isTestBuild }}" -on: - repository_dispatch: - types: ['linux-source-verification:*'] - -# Input: -# buildSha: The SHA of the commit to build, e.g. of the branch or -# refs/pull/1234/head for PR -# branch: The branch to build, for a PR in the form `PR-1234` -# baseBranch: For a PR the base branch, otherwise the same as `branch` -# baseRef: The ref of the previous commit. For a PR the same as `baseBranch`. -# user: The user that triggered the build or created the PR -# isTestBuild: false for Releases, otherwise true - -env: - STATUS_CONTEXT: 'Linux source verification' - GH_TOKEN: ${{ github.token }} - -jobs: - source_verification: - name: Verify source tarball and Linux source package - runs-on: ubuntu-latest - outputs: - KEYMAN_VERSION: ${{ steps.version_step.outputs.KEYMAN_VERSION }} - - steps: - - name: Set pending status on PR builds - id: set_status - if: github.event.client_payload.isTestBuild == 'true' - shell: bash - run: | - gh api \ - --method POST \ - -H "Accept: application/vnd.github+json" \ - /repos/$GITHUB_REPOSITORY/statuses/${{ github.event.client_payload.buildSha }} \ - -f state='pending' \ - -f target_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \ - -f description='Linux source verification started' \ - -f context="$STATUS_CONTEXT" - - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - ref: '${{ github.event.client_payload.buildSha }}' - fetch-depth: 0 - clean: false - path: ${{ github.workspace }}/keyman/ - - - name: Install devscripts - uses: ./keyman/.github/actions/apt-install - with: - packages: devscripts equivs - - - name: Install dependencies - run: | - cd ${{ github.workspace }}/keyman/linux - ./scripts/deb-packaging.sh --gha dependencies - - - name: Set version as output parameter - id: version_step - shell: bash - run: | - THIS_SCRIPT="${{ github.workspace }}/keyman/.github/workflows/linux-source-verification.yml" - if [[ -f "${THIS_SCRIPT%/*}/../../resources/build/builder-basic.inc.sh" ]]; then - . "${THIS_SCRIPT%/*}/../../resources/build/builder-basic.inc.sh" - else - # TODO: remove following block post v19 release (file was renamed in v19 alpha) - if [[ -f "${THIS_SCRIPT%/*}/../../resources/build/build-utils.sh" ]]; then - . "${THIS_SCRIPT%/*}/../../resources/build/build-utils.sh" - fi - fi - echo "KEYMAN_VERSION=${KEYMAN_VERSION:-${VERSION}}" >> $GITHUB_OUTPUT - - - name: Verify source tarball - shell: bash - run: | - cd ${{ github.workspace }}/keyman/linux - ./scripts/verify_source.sh --source-only "${{ github.workspace }}/keyman-${{ steps.version_step.outputs.KEYMAN_VERSION }}" - - - name: Verify Ubuntu source package - shell: bash - run: | - cd ${{ github.workspace }}/keyman/linux - ./scripts/verify_source.sh --no-create-tarball --launchpad-only "${{ github.workspace }}/keyman-${{ steps.version_step.outputs.KEYMAN_VERSION }}" - - set_status: - name: Set result status on PR builds - needs: [source_verification] - runs-on: ubuntu-latest - if: ${{ always() && github.event.client_payload.isTestBuild == 'true' }} - steps: - - name: Set success - if: needs.source_verification.result == 'success' - run: | - echo "RESULT=success" >> $GITHUB_ENV - echo "MSG=Linux source verification succeeded" >> $GITHUB_ENV - - - name: Set cancelled - if: needs.source_verification.result == 'cancelled' - run: | - echo "RESULT=error" >> $GITHUB_ENV - echo "MSG=Linux source verification cancelled" >> $GITHUB_ENV - - - name: Set failure - if: needs.source_verification.result == 'failure' - run: | - echo "RESULT=failure" >> $GITHUB_ENV - echo "MSG=Linux source verification failed" >> $GITHUB_ENV - - - name: Set final status - run: | - gh api \ - --method POST \ - -H "Accept: application/vnd.github+json" \ - /repos/$GITHUB_REPOSITORY/statuses/${{ github.event.client_payload.buildSha }} \ - -f state="$RESULT" \ - -f target_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \ - -f description="$MSG" \ - -f context="$STATUS_CONTEXT" diff --git a/.github/workflows/pr-build-status.yml b/.github/workflows/pr-build-status.yml index f2f0382556..0e32214ed3 100644 --- a/.github/workflows/pr-build-status.yml +++ b/.github/workflows/pr-build-status.yml @@ -78,8 +78,6 @@ jobs: summary += addStatus(o, 'user-test', status.context, status.state); } else if(status.context == 'API Verification') { summary += addStatus(o, 'check', status.context, status.state); - } else if (status.context == 'Linux source verification') { - summary += addStatus(o, 'check', status.context, status.state); } else if(status.context == 'Ubuntu Packaging') { summary += addStatus(o, 'build', status.context, status.state); } else if(status.context == 'npm pack/publish') { diff --git a/docs/linux/packaging.md b/docs/linux/packaging.md index 0b493cc1d6..c7303ebced 100644 --- a/docs/linux/packaging.md +++ b/docs/linux/packaging.md @@ -184,28 +184,14 @@ To upload the packages to launchpad, run the following script from the `linux/` ./scripts/launchpad.sh [options] ``` -#### Parameters - -- `--no-download` - Don't download source. Assume keyman- exists - as subdirectory of current dir. -- `--upload` - Upload to launchpad. If omitted only simulate the upload. -- `--no-upload` - Don't upload to launchpad, don't even simulate it. -- `--no-lintian` - Don't run lintian while creating soure package. -- `--project ""` - only upload this package -- `--dist ""` - only upload for this distribution -- `--packageversion ""` - Default `1~sil1`. Normally use the - default so don't specify it. But if you change packaging and run another - upload you need to increment the number at the end of `version`, e.g. - next one is `1~sil2` then `1~sil3`… -- `--outputdir ` - Directory for resulting artifacts. - Default: `$KEYMAN_ROOT/linux/launchpad`. +See `launchpad.sh --help` for supported parameters. ### Releasing a new version As part of releasing a new version it might be good to do some local testing first before uploading to Launchpad: -- Run `launchpad.sh` without `--upload` to build the packages +- Run `launchpad.sh` with `--simulate` to build the packages - Then install them on a clean VM and make sure no glaring bugs - Once you are happy with the packages you can run `launchpad.sh` with `--upload` diff --git a/linux/scripts/launchpad.sh b/linux/scripts/launchpad.sh index b2c2f650a4..30c7ad61dc 100755 --- a/linux/scripts/launchpad.sh +++ b/linux/scripts/launchpad.sh @@ -15,9 +15,9 @@ builder_describe \ "Build source packages from nightly builds and upload to PPA" \ build \ "--no-download Don't download source. Assume keyman- exists as subdirectory of current dir." \ - "--upload Upload to launchpad. If omitted only simulate the upload." \ - "--no-upload Don't upload to launchpad, don't even simulate it." \ - "--no-lintian Don't run lintian while creating soure package." \ + "--upload Upload to launchpad." \ + "--simulate Simulate the upload to launchpad." \ + "--no-lintian Don't run lintian while creating source package." \ "--project=PROJECT Only upload this project. Default: keyman" \ "--dist=DIST Only upload this distribution. Default: upload all supported dists." \ "--packageversion=PACKAGEVERSION String to append to the package version. Default: '1~sil1'." \ @@ -33,10 +33,10 @@ if [[ -z "${OUTPUTDIR:-}" ]]; then OUTPUTDIR="${KEYMAN_ROOT}/linux/launchpad" fi -if builder_has_option --upload; then - SIM="" -else +if builder_has_option --simulate; then SIM="-s" +else + SIM="" fi if builder_has_option --no-lintian; then @@ -80,7 +80,7 @@ for proj in ${projects:-}; do # shellcheck disable=SC2248 # no quotes for $LINTIAN_OPTS - might be empty string debuild ${LINTIAN_OPTS} -d -S -sa -Zxz done - if ! builder_has_option --no-upload; then + if builder_has_option --upload || builder_has_option --simulate; then cd .. for dist in ${distributions}; do # shellcheck disable=SC2248 # no quotes for $SIM - it might not be set diff --git a/linux/scripts/package-build.inc.sh b/linux/scripts/package-build.inc.sh index 44995db98d..852983db9f 100644 --- a/linux/scripts/package-build.inc.sh +++ b/linux/scripts/package-build.inc.sh @@ -14,7 +14,7 @@ function checkPrerequisites() { exit 1 fi - if ! command -v xmllint > /dev/null; then + if ! builder_has_option --no-download && ! command -v xmllint > /dev/null; then echo "you must install xmllint (libxml2-utils package) to use this script" exit 1 fi diff --git a/linux/scripts/verify_source.sh b/linux/scripts/verify_source.sh index 16951f6879..52d9b5bd00 100755 --- a/linux/scripts/verify_source.sh +++ b/linux/scripts/verify_source.sh @@ -7,38 +7,23 @@ ## START STANDARD BUILD SCRIPT INCLUDE # adjust relative paths as necessary THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" -. "${THIS_SCRIPT%/*}/../../resources/build/builder-basic.inc.sh" +# shellcheck source=resources/build/builder-full.inc.sh +. "${THIS_SCRIPT%/*}/../../resources/build/builder-full.inc.sh" ## END STANDARD BUILD SCRIPT INCLUDE -VERIFY_SOURCE=true -VERIFY_LAUNCHPAD=true -CREATE_TARBALL=true +builder_describe \ + "Verify source tarball and source package" \ + build \ + "--no-create-tarball Skip creating a new tarball. Instead use existing one." \ + "--source-only Only check that it's possible to build Keyman from the source tarball." \ + "--launchpad-only Only check that lintian doesn't produce errors when run on the source package." \ + "--target-dir=TARGET_DIR Directory where to put generate files. Default: /tmp/keyman-." -while (( $# > 0)); do - case "$1" in - --no-create-tarball) - CREATE_TARBALL=false - shift - ;; - --source-only) - VERIFY_LAUNCHPAD=false - shift - ;; - --launchpad-only) - VERIFY_SOURCE=false - shift - ;; - --help|-h) - echo "Usage: $0 [--source-only|--launchpad-only] [--no-create-tarball] []" - exit 0 - ;; - *) - break - ;; - esac -done +builder_parse "$@" -TARGET_DIR="${1:-/tmp/keyman-${KEYMAN_VERSION}}" +if ! builder_has_option --target-dir; then + TARGET_DIR="/tmp/keyman-${KEYMAN_VERSION}" +fi create_source_tarball() { local target_dir="$1" @@ -83,7 +68,7 @@ create_source_package() { cp -r "../keyman" "keyman-${KEYMAN_VERSION}" cp -r "${KEYMAN_ROOT}/linux/debian" "keyman-${KEYMAN_VERSION}" cp "${target_dir}/keyman-${KEYMAN_VERSION}.tar.xz" "keyman_${KEYMAN_VERSION}.orig.tar.xz" - "keyman-${KEYMAN_VERSION}/linux/scripts/launchpad.sh" --no-download --no-upload \ + "keyman-${KEYMAN_VERSION}/linux/scripts/launchpad.sh" --no-download \ --dist "$(lsb_release -c -s)" --outputdir "${target_dir}/launchpad" --no-lintian } @@ -94,13 +79,13 @@ verify_lintian() { cd "${KEYMAN_ROOT}/linux" -if ${CREATE_TARBALL}; then +if ! builder_has_option --no-create-tarball; then builder_echo start tarball "Building source tarball" create_source_tarball "${TARGET_DIR}" builder_echo end tarball success "Finished building source tarball" fi -if ${VERIFY_SOURCE}; then +if ! builder_has_option --launchpad-only; then builder_echo start verifySource "Verifying source tarball" extract_source_tarball "${TARGET_DIR}" verify_can_build "${TARGET_DIR}/keyman" @@ -108,7 +93,7 @@ if ${VERIFY_SOURCE}; then builder_echo end verifySource success "Finished verifying source tarball" fi -if ${VERIFY_LAUNCHPAD}; then +if ! builder_has_option --source-only; then builder_echo start lintian "Verifying Launchpad source package" extract_source_tarball "${TARGET_DIR}" create_source_package "${TARGET_DIR}" diff --git a/resources/build/ci/trigger-definitions.inc.sh b/resources/build/ci/trigger-definitions.inc.sh index 1b27ce137a..bac1cf99de 100644 --- a/resources/build/ci/trigger-definitions.inc.sh +++ b/resources/build/ci/trigger-definitions.inc.sh @@ -85,7 +85,7 @@ bc_test_all=() bc_test_android=(KeymanAndroid_TestPullRequests KeymanAndroid_TestSamplesAndTestProjects) bc_test_ios=(Keyman_iOS_TestPullRequests Keyman_iOS_TestSamplesAndTestProjects) -bc_test_linux=(KeymanLinux_TestPullRequests Keyman_Linux_Test_Integration Keyman_Common_KPAPI_TestPullRequests_Linux deb-pr-packaging_GitHub linux-source-verification_GitHub) +bc_test_linux=(KeymanLinux_TestPullRequests Keyman_Linux_Test_Integration Keyman_Common_KPAPI_TestPullRequests_Linux deb-pr-packaging_GitHub) bc_test_mac=(Keyman_KeymanMac_PullRequests Keyman_Common_KPAPI_TestPullRequests_macOS) bc_test_windows=(KeymanDesktop_TestPullRequests KeymanDesktop_TestPrRenderOnScreenKeyboards Keyman_Common_KPAPI_TestPullRequests_Windows core-arm64-windows-test_GitHub) bc_test_web=(Keymanweb_TestPullRequests Keyman_Common_KPAPI_TestPullRequests_WASM) diff --git a/resources/build/pr-build-status/pr-build-status.mjs b/resources/build/pr-build-status/pr-build-status.mjs index 31fc3cab38..06ac26fc09 100644 --- a/resources/build/pr-build-status/pr-build-status.mjs +++ b/resources/build/pr-build-status/pr-build-status.mjs @@ -50,8 +50,6 @@ summary += addStatus(o, 'user-test', status.context, status.state); } else if(status.context == 'API Verification') { summary += addStatus(o, 'check', status.context, status.state); - } else if (status.context == 'Linux source verification') { - summary += addStatus(o, 'check', status.context, status.state); } else if(status.context == 'Ubuntu Packaging') { summary += addStatus(o, 'build', status.context, status.state); } else if(status.context == 'npm pack/publish') {