chore(linux): address code review comments

- make `verify_source.sh` a full builder script
- make options for `launchpad.sh` less confusing
- reference `--help` in `packaging.md`
- run new script as part of Ubuntu packaging workflow

Co-authored-by: Marc Durdin <marc@durdin.net>

.github/workflows/deb-packaging.yml

@@ -29,6 +29,7 @@ env:
   COLOR_GREEN: "\e[32m"
   GH_TOKEN: ${{ github.token }}
   STATUS_CONTEXT: 'Ubuntu Packaging'
+  STATUS_CONTEXT_SOURCE_VERIFICATION: 'Linux source verification'
   DEBFULLNAME: 'Keyman GHA packager'
   DEBEMAIL: 'support@keyman.com'
   _builder_timings: false
@@ -403,3 +404,92 @@ jobs:
           -f target_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \
           -f description="$MSG" \
           -f context="$STATUS_CONTEXT"
+
+  #---------------------------------------------------------------------------------------------
+  source_verification:
+    name: Verify source tarball and Linux source package
+    if: github.repository == 'keymanapp/keyman' || github.event.client_payload.force
+    needs: sourcepackage
+    runs-on: ubuntu-latest
+    outputs:
+      KEYMAN_VERSION: ${{ needs.sourcepackage.outputs.KEYMAN_VERSION }}
+
+    steps:
+    - name: Set pending status on PR builds
+      id: set_status
+      if: github.event.client_payload.isTestBuild == 'true'
+      shell: bash
+      run: |
+        gh api \
+          --method POST \
+          -H "Accept: application/vnd.github+json" \
+          /repos/$GITHUB_REPOSITORY/statuses/${{ github.event.client_payload.buildSha }} \
+          -f state='pending' \
+          -f target_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \
+          -f description='Linux source verification started' \
+          -f context="$STATUS_CONTEXT_SOURCE_VERIFICATION"
+
+    - name: Checkout
+      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      with:
+        ref: '${{ github.event.client_payload.buildSha }}'
+        fetch-depth: 0
+        clean: false
+        path: ${{ github.workspace }}/keyman/
+
+    - name: Install devscripts
+      uses: ./keyman/.github/actions/apt-install
+      with:
+        packages: devscripts equivs
+
+    - name: Install dependencies
+      run: |
+        cd ${{ github.workspace }}/keyman/linux
+        ./scripts/deb-packaging.sh --gha dependencies
+
+    - name: Verify source tarball
+      shell: bash
+      run: |
+        cd ${{ github.workspace }}/keyman/linux
+        ./scripts/verify_source.sh --source-only --target-dir "${{ github.workspace }}/keyman-${{ needs.sourcepackage.outputs.KEYMAN_VERSION }}"
+
+    - name: Verify Ubuntu source package
+      shell: bash
+      run: |
+        cd ${{ github.workspace }}/keyman/linux
+        ./scripts/verify_source.sh --no-create-tarball --launchpad-only --target-dir "${{ github.workspace }}/keyman-${{ needs.sourcepackage.outputs.KEYMAN_VERSION }}"
+
+  set_status_source_verification:
+    name: Set result status of source verification on PR builds
+    needs: [source_verification]
+    runs-on: ubuntu-latest
+    if: ${{ always() && github.event.client_payload.isTestBuild == 'true' }}
+    steps:
+    - name: Set success
+      if: needs.source_verification.result == 'success'
+      run: |
+        echo "RESULT=success" >> $GITHUB_ENV
+        echo "MSG=Linux source verification succeeded" >> $GITHUB_ENV
+
+    - name: Set cancelled
+      if: needs.source_verification.result == 'cancelled'
+      run: |
+        echo "RESULT=error" >> $GITHUB_ENV
+        echo "MSG=Linux source verification cancelled" >> $GITHUB_ENV
+
+    - name: Set failure
+      if: needs.source_verification.result == 'failure'
+      run: |
+        echo "RESULT=failure" >> $GITHUB_ENV
+        echo "MSG=Linux source verification failed" >> $GITHUB_ENV
+
+    - name: Set final status
+      run: |
+        gh api \
+          --method POST \
+          -H "Accept: application/vnd.github+json" \
+          /repos/$GITHUB_REPOSITORY/statuses/${{ github.event.client_payload.buildSha }} \
+          -f state="$RESULT" \
+          -f target_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \
+          -f description="$MSG" \
+          -f context="$STATUS_CONTEXT_SOURCE_VERIFICATION"

.github/workflows/linux-source-verification.yml

@@ -1,128 +0,0 @@
-# Keyman is copyright (C) SIL Global. MIT License.
-#
-# This workflow verifies two things:
-# - the source tarball that gets uploaded to downloads.keyman.com. It
-#   verifies that it contains all files needed to run `build.sh`
-# - the source package that gets uploaded to launchpad. It verifies that
-#   running lintian doesn't output any errors.
-
-name: "Linux source verification"
-run-name: "Linux source verification - ${{ github.event.client_payload.branch }} (branch ${{ github.event.client_payload.baseBranch }}), by @${{ github.event.client_payload.user }}, testbuild: ${{ github.event.client_payload.isTestBuild }}"
-on:
-  repository_dispatch:
-    types: ['linux-source-verification:*']
-
-# Input:
-# buildSha:    The SHA of the commit to build, e.g. of the branch or
-#              refs/pull/1234/head for PR
-# branch:      The branch to build, for a PR in the form `PR-1234`
-# baseBranch:  For a PR the base branch, otherwise the same as `branch`
-# baseRef:     The ref of the previous commit. For a PR the same as `baseBranch`.
-# user:        The user that triggered the build or created the PR
-# isTestBuild: false for Releases, otherwise true
-
-env:
-  STATUS_CONTEXT: 'Linux source verification'
-  GH_TOKEN: ${{ github.token }}
-
-jobs:
-  source_verification:
-    name: Verify source tarball and Linux source package
-    runs-on: ubuntu-latest
-    outputs:
-      KEYMAN_VERSION: ${{ steps.version_step.outputs.KEYMAN_VERSION }}
-
-    steps:
-    - name: Set pending status on PR builds
-      id: set_status
-      if: github.event.client_payload.isTestBuild == 'true'
-      shell: bash
-      run: |
-        gh api \
-          --method POST \
-          -H "Accept: application/vnd.github+json" \
-          /repos/$GITHUB_REPOSITORY/statuses/${{ github.event.client_payload.buildSha }} \
-          -f state='pending' \
-          -f target_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \
-          -f description='Linux source verification started' \
-          -f context="$STATUS_CONTEXT"
-
-    - name: Checkout
-      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-      with:
-        ref: '${{ github.event.client_payload.buildSha }}'
-        fetch-depth: 0
-        clean: false
-        path: ${{ github.workspace }}/keyman/
-
-    - name: Install devscripts
-      uses: ./keyman/.github/actions/apt-install
-      with:
-        packages: devscripts equivs
-
-    - name: Install dependencies
-      run: |
-        cd ${{ github.workspace }}/keyman/linux
-        ./scripts/deb-packaging.sh --gha dependencies
-
-    - name: Set version as output parameter
-      id: version_step
-      shell: bash
-      run: |
-        THIS_SCRIPT="${{ github.workspace }}/keyman/.github/workflows/linux-source-verification.yml"
-        if [[ -f "${THIS_SCRIPT%/*}/../../resources/build/builder-basic.inc.sh" ]]; then
-          . "${THIS_SCRIPT%/*}/../../resources/build/builder-basic.inc.sh"
-        else
-          # TODO: remove following block post v19 release (file was renamed in v19 alpha)
-          if [[ -f "${THIS_SCRIPT%/*}/../../resources/build/build-utils.sh" ]]; then
-            . "${THIS_SCRIPT%/*}/../../resources/build/build-utils.sh"
-          fi
-        fi
-        echo "KEYMAN_VERSION=${KEYMAN_VERSION:-${VERSION}}" >> $GITHUB_OUTPUT
-
-    - name: Verify source tarball
-      shell: bash
-      run: |
-        cd ${{ github.workspace }}/keyman/linux
-        ./scripts/verify_source.sh --source-only "${{ github.workspace }}/keyman-${{ steps.version_step.outputs.KEYMAN_VERSION }}"
-
-    - name: Verify Ubuntu source package
-      shell: bash
-      run: |
-        cd ${{ github.workspace }}/keyman/linux
-        ./scripts/verify_source.sh --no-create-tarball --launchpad-only "${{ github.workspace }}/keyman-${{ steps.version_step.outputs.KEYMAN_VERSION }}"
-
-  set_status:
-    name: Set result status on PR builds
-    needs: [source_verification]
-    runs-on: ubuntu-latest
-    if: ${{ always() && github.event.client_payload.isTestBuild == 'true' }}
-    steps:
-    - name: Set success
-      if: needs.source_verification.result == 'success'
-      run: |
-        echo "RESULT=success" >> $GITHUB_ENV
-        echo "MSG=Linux source verification succeeded" >> $GITHUB_ENV
-
-    - name: Set cancelled
-      if: needs.source_verification.result == 'cancelled'
-      run: |
-        echo "RESULT=error" >> $GITHUB_ENV
-        echo "MSG=Linux source verification cancelled" >> $GITHUB_ENV
-
-    - name: Set failure
-      if: needs.source_verification.result == 'failure'
-      run: |
-        echo "RESULT=failure" >> $GITHUB_ENV
-        echo "MSG=Linux source verification failed" >> $GITHUB_ENV
-
-    - name: Set final status
-      run: |
-        gh api \
-          --method POST \
-          -H "Accept: application/vnd.github+json" \
-          /repos/$GITHUB_REPOSITORY/statuses/${{ github.event.client_payload.buildSha }} \
-          -f state="$RESULT" \
-          -f target_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \
-          -f description="$MSG" \
-          -f context="$STATUS_CONTEXT"

.github/workflows/pr-build-status.yml

@@ -78,8 +78,6 @@ jobs:
                   summary += addStatus(o, 'user-test', status.context, status.state);
                 } else if(status.context == 'API Verification') {
                   summary += addStatus(o, 'check', status.context, status.state);
-                } else if (status.context == 'Linux source verification') {
-                  summary += addStatus(o, 'check', status.context, status.state);
                 } else if(status.context == 'Ubuntu Packaging') {
                   summary += addStatus(o, 'build', status.context, status.state);
                 } else if(status.context == 'npm pack/publish') {

docs/linux/packaging.md

@@ -184,28 +184,14 @@ To upload the packages to launchpad, run the following script from the `linux/`
 ./scripts/launchpad.sh [options]
 ```
 
-#### Parameters
-
-- `--no-download` - Don't download source. Assume keyman-<version> exists
-  as subdirectory of current dir.
-- `--upload` - Upload to launchpad. If omitted only simulate the upload.
-- `--no-upload` - Don't upload to launchpad, don't even simulate it.
-- `--no-lintian` - Don't run lintian while creating soure package.
-- `--project "<project>"` - only upload this package
-- `--dist "<dist>"` - only upload for this distribution
-- `--packageversion "<version>"` - Default `1~sil1`. Normally use the
-  default so don't specify it. But if you change packaging and run another
-  upload you need to increment the number at the end of `version`, e.g.
-  next one is `1~sil2` then `1~sil3`…
-- `--outputdir <outputdir>` - Directory for resulting artifacts.
-  Default: `$KEYMAN_ROOT/linux/launchpad`.
+See `launchpad.sh --help` for supported parameters.
 
 ### Releasing a new version
 
 As part of releasing a new version it might be good to do some local testing
 first before uploading to Launchpad:
 
-- Run `launchpad.sh` without `--upload` to build the packages
+- Run `launchpad.sh` with `--simulate` to build the packages
 - Then install them on a clean VM and make sure no glaring bugs
 - Once you are happy with the packages you can run `launchpad.sh` with `--upload`
 

linux/scripts/launchpad.sh

@@ -15,9 +15,9 @@ builder_describe \
   "Build source packages from nightly builds and upload to PPA" \
   build \
   "--no-download                    Don't download source. Assume keyman-<version> exists as subdirectory of current dir." \
-  "--upload                         Upload to launchpad. If omitted only simulate the upload." \
-  "--no-upload                      Don't upload to launchpad, don't even simulate it." \
-  "--no-lintian                     Don't run lintian while creating soure package." \
+  "--upload                         Upload to launchpad." \
+  "--simulate                       Simulate the upload to launchpad." \
+  "--no-lintian                     Don't run lintian while creating source package." \
   "--project=PROJECT                Only upload this project. Default: keyman" \
   "--dist=DIST                      Only upload this distribution. Default: upload all supported dists." \
   "--packageversion=PACKAGEVERSION  String to append to the package version. Default: '1~sil1'." \
@@ -33,10 +33,10 @@ if [[ -z "${OUTPUTDIR:-}" ]]; then
   OUTPUTDIR="${KEYMAN_ROOT}/linux/launchpad"
 fi
 
-if builder_has_option --upload; then
-  SIM=""
-else
+if builder_has_option --simulate; then
   SIM="-s"
+else
+  SIM=""
 fi
 
 if builder_has_option --no-lintian; then
@@ -80,7 +80,7 @@ for proj in ${projects:-}; do
     # shellcheck disable=SC2248  # no quotes for $LINTIAN_OPTS - might be empty string
     debuild ${LINTIAN_OPTS} -d -S -sa -Zxz
   done
-  if ! builder_has_option --no-upload; then
+  if builder_has_option --upload || builder_has_option --simulate; then
     cd ..
     for dist in ${distributions}; do
       # shellcheck disable=SC2248  # no quotes for $SIM - it might not be set

linux/scripts/package-build.inc.sh

@@ -14,7 +14,7 @@ function checkPrerequisites() {
     exit 1
   fi
 
-  if ! command -v xmllint > /dev/null; then
+  if ! builder_has_option --no-download && ! command -v xmllint > /dev/null; then
     echo "you must install xmllint (libxml2-utils package) to use this script"
     exit 1
   fi

linux/scripts/verify_source.sh

@@ -7,38 +7,23 @@
 ## START STANDARD BUILD SCRIPT INCLUDE
 # adjust relative paths as necessary
 THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")"
-. "${THIS_SCRIPT%/*}/../../resources/build/builder-basic.inc.sh"
+# shellcheck source=resources/build/builder-full.inc.sh
+. "${THIS_SCRIPT%/*}/../../resources/build/builder-full.inc.sh"
 ## END STANDARD BUILD SCRIPT INCLUDE
 
-VERIFY_SOURCE=true
-VERIFY_LAUNCHPAD=true
-CREATE_TARBALL=true
+builder_describe \
+  "Verify source tarball and source package" \
+  build \
+  "--no-create-tarball   Skip creating a new tarball. Instead use existing one." \
+  "--source-only         Only check that it's possible to build Keyman from the source tarball." \
+  "--launchpad-only      Only check that lintian doesn't produce errors when run on the source package." \
+  "--target-dir=TARGET_DIR  Directory where to put generate files. Default: /tmp/keyman-<version>."
 
-while (( $# > 0)); do
-  case "$1" in
-    --no-create-tarball)
-      CREATE_TARBALL=false
-      shift
-      ;;
-    --source-only)
-      VERIFY_LAUNCHPAD=false
-      shift
-      ;;
-    --launchpad-only)
-      VERIFY_SOURCE=false
-      shift
-      ;;
-    --help|-h)
-      echo "Usage: $0 [--source-only|--launchpad-only] [--no-create-tarball] [<target directory>]"
-      exit 0
-      ;;
-    *)
-      break
-      ;;
-  esac
-done
+builder_parse "$@"
 
-TARGET_DIR="${1:-/tmp/keyman-${KEYMAN_VERSION}}"
+if ! builder_has_option --target-dir; then
+  TARGET_DIR="/tmp/keyman-${KEYMAN_VERSION}"
+fi
 
 create_source_tarball() {
   local target_dir="$1"
@@ -83,7 +68,7 @@ create_source_package() {
   cp -r "../keyman" "keyman-${KEYMAN_VERSION}"
   cp -r "${KEYMAN_ROOT}/linux/debian" "keyman-${KEYMAN_VERSION}"
   cp "${target_dir}/keyman-${KEYMAN_VERSION}.tar.xz" "keyman_${KEYMAN_VERSION}.orig.tar.xz"
-  "keyman-${KEYMAN_VERSION}/linux/scripts/launchpad.sh" --no-download --no-upload \
+  "keyman-${KEYMAN_VERSION}/linux/scripts/launchpad.sh" --no-download \
     --dist "$(lsb_release -c -s)" --outputdir "${target_dir}/launchpad" --no-lintian
 }
 
@@ -94,13 +79,13 @@ verify_lintian() {
 
 cd "${KEYMAN_ROOT}/linux"
 
-if ${CREATE_TARBALL}; then
+if ! builder_has_option --no-create-tarball; then
   builder_echo start tarball "Building source tarball"
   create_source_tarball "${TARGET_DIR}"
   builder_echo end tarball success "Finished building source tarball"
 fi
 
-if ${VERIFY_SOURCE}; then
+if ! builder_has_option --launchpad-only; then
   builder_echo start verifySource "Verifying source tarball"
   extract_source_tarball "${TARGET_DIR}"
   verify_can_build "${TARGET_DIR}/keyman"
@@ -108,7 +93,7 @@ if ${VERIFY_SOURCE}; then
   builder_echo end verifySource success "Finished verifying source tarball"
 fi
 
-if ${VERIFY_LAUNCHPAD}; then
+if ! builder_has_option --source-only; then
   builder_echo start lintian "Verifying Launchpad source package"
   extract_source_tarball "${TARGET_DIR}"
   create_source_package "${TARGET_DIR}"

resources/build/ci/trigger-definitions.inc.sh

@@ -85,7 +85,7 @@ bc_test_all=()
 
 bc_test_android=(KeymanAndroid_TestPullRequests KeymanAndroid_TestSamplesAndTestProjects)
 bc_test_ios=(Keyman_iOS_TestPullRequests Keyman_iOS_TestSamplesAndTestProjects)
-bc_test_linux=(KeymanLinux_TestPullRequests Keyman_Linux_Test_Integration Keyman_Common_KPAPI_TestPullRequests_Linux deb-pr-packaging_GitHub linux-source-verification_GitHub)
+bc_test_linux=(KeymanLinux_TestPullRequests Keyman_Linux_Test_Integration Keyman_Common_KPAPI_TestPullRequests_Linux deb-pr-packaging_GitHub)
 bc_test_mac=(Keyman_KeymanMac_PullRequests Keyman_Common_KPAPI_TestPullRequests_macOS)
 bc_test_windows=(KeymanDesktop_TestPullRequests KeymanDesktop_TestPrRenderOnScreenKeyboards Keyman_Common_KPAPI_TestPullRequests_Windows core-arm64-windows-test_GitHub)
 bc_test_web=(Keymanweb_TestPullRequests Keyman_Common_KPAPI_TestPullRequests_WASM)

resources/build/pr-build-status/pr-build-status.mjs

@@ -50,8 +50,6 @@
                   summary += addStatus(o, 'user-test', status.context, status.state);
                 } else if(status.context == 'API Verification') {
                   summary += addStatus(o, 'check', status.context, status.state);
-                } else if (status.context == 'Linux source verification') {
-                  summary += addStatus(o, 'check', status.context, status.state);
                 } else if(status.context == 'Ubuntu Packaging') {
                   summary += addStatus(o, 'build', status.context, status.state);
                 } else if(status.context == 'npm pack/publish') {