diff --git a/.github/actions/check_cirrus_cron/cron_failures.sh b/.github/actions/check_cirrus_cron/cron_failures.sh deleted file mode 100755 index e97754d629..0000000000 --- a/.github/actions/check_cirrus_cron/cron_failures.sh +++ /dev/null @@ -1,92 +0,0 @@ -#!/bin/bash - -set -eo pipefail - -# Intended to be executed from a github action workflow step. -# Outputs the Cirrus cron names and IDs of any failed builds - -source $(dirname "${BASH_SOURCE[0]}")/lib.sh - -_errfmt="Expecting %s value to not be empty" -if [[ -z "$GITHUB_REPOSITORY" ]]; then # / - err $(printf "$_errfmt" "\$GITHUB_REPOSITORY") -elif [[ -z "$ID_NAME_FILEPATH" ]]; then # output filepath - err $(printf "$_errfmt" "\$ID_NAME_FILEPATH") -fi - -confirm_gha_environment - -mkdir -p ./artifacts -cat > ./artifacts/query_raw.json << "EOF" -query { - ownerRepository(platform: "github", owner: "@@OWNER@@", name: "@@REPO@@") { - cronSettings { - name - lastInvocationBuild { - id - status - } - } - } -} -EOF -# Makes for easier copy/pasting query to/from -# https://cirrus-ci.com/explorer -owner=$(cut -d '/' -f 1 <<<"$GITHUB_REPOSITORY") -repo=$(cut -d '/' -f 2 <<<"$GITHUB_REPOSITORY") -sed -r -e "s/@@OWNER@@/$owner/g" -e "s/@@REPO@@/$repo/g" \ - ./artifacts/query_raw.json > ./artifacts/query.json - -if grep -q '@@' ./artifacts/query.json; then - err "Found unreplaced substitution token in query JSON" -fi - -# The query should never ever return an empty-list, unless there are no cirrus-cron -# jobs defined for the repository. In that case, this monitoring script shouldn't -# be running anyway. -filt_head='.data.ownerRepository.cronSettings' - -gql "$(<./artifacts/query.json)" "$filt_head" > ./artifacts/reply.json -# e.x. reply.json -# { -# "data": { -# "ownerRepository": { -# "cronSettings": [ -# { -# "name": "Keepalive_v2.0", -# "lastInvocationBuild": { -# "id": "5776050544181248", -# "status": "EXECUTING" -# } -# }, -# { -# "name": "Keepalive_v1.9", -# "lastInvocationBuild": { -# "id": "5962921081569280", -# "status": "COMPLETED" -# } -# }, -# { -# "name": "Keepalive_v2.0.5-rhel", -# "lastInvocationBuild": { -# "id": "5003065549914112", -# "status": "FAILED" -# } -# ... - -# Output format: -# Where may contain multiple words -filt="$filt_head | map(select(.lastInvocationBuild.status==\"FAILED\") | {id:.lastInvocationBuild.id, name:.name} | join(\" \")) | join(\"\n\")" -jq --raw-output "$filt" ./artifacts/reply.json > "$ID_NAME_FILEPATH" - -# Print out the file to assist in job debugging -echo " " -cat "$ID_NAME_FILEPATH" - -# Count non-empty lines (in case there are any) -records=$(awk -r -e '/\w+/{print $0}' "$ID_NAME_FILEPATH" | wc -l) -# Set the output of this step. -# Ref: https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-output-parameter -# shellcheck disable=SC2154 -echo "failures=$records" >> $GITHUB_OUTPUT -echo "Total failed Cirrus-CI cron builds: $records" diff --git a/.github/actions/check_cirrus_cron/lib.sh b/.github/actions/check_cirrus_cron/lib.sh deleted file mode 100644 index 7e8d42e83b..0000000000 --- a/.github/actions/check_cirrus_cron/lib.sh +++ /dev/null @@ -1,95 +0,0 @@ - - -# Send text to stderr -msg() { - echo "$@" > /dev/stderr -} - -# Must be called from top-level of script, not another function. -err() { - # Ref: https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions - msg "::error file=${BASH_SOURCE[1]},line=${BASH_LINENO[0]}::$*" - exit 1 -} - -confirm_gha_environment() { - local _err_fmt - _err_fmt="I don't seem to be running from a github-actions workflow" - # These are all defined by github-actions - # shellcheck disable=SC2154 - if [[ -z "$GITHUB_OUTPUT" ]]; then - err "$_err_fmt, \$GITHUB_OUTPUT is empty" - elif [[ -z "$GITHUB_WORKFLOW" ]]; then - err "$_err_fmt, \$GITHUB_WORKFLOW is empty" - elif [[ ! -d "$GITHUB_WORKSPACE" ]]; then - # Defined by github-actions - # shellcheck disable=SC2154 - err "$_err_fmt, \$GITHUB_WORKSPACE='$GITHUB_WORKSPACE' isn't a directory" - fi - - cd "$GITHUB_WORKSPACE" || false -} - -# Using python3 here is a compromise for readability and -# properly handling quote, control and unicode character encoding. -escape_query() { - local json_string - # Assume it's okay to squash repeated whitespaces inside the query - json_string=$(printf '%s' "$1" | \ - tr --delete '\r\n' | \ - tr --squeeze-repeats '[[:space:]]' | \ - python3 -c 'import sys,json; print(json.dumps(sys.stdin.read()))') - # The $json_string in message is already quoted - echo -n "$json_string" -} - -# Given a GraphQL query/mutation, fire it at the API. -# and return the output on stdout. The optional -# second parameter may contain a jq filter-string. -# When provided, if the GQL result is empty, null, -# fails to parse, or does not match the filter-string, -# non-zero will be returned. -gql() { - local e_query query - e_query=$(escape_query "$1") - query="{\"query\": $e_query}" - local filter - filter="$2" - local output - local filtered - msg "::group::Posting GraphQL Query and checking result" - msg "query: " - if ! jq -e . <<<"$query" > /dev/stderr; then - msg "::error file=${BASH_SOURCE[1]},line=${BASH_LINENO[0]}::Invalid query JSON: $query" - return 1 - fi - # SECRET_CIRRUS_API_KEY is defined github secret - # shellcheck disable=SC2154 - if output=$(curl \ - --request POST \ - --silent \ - --show-error \ - --location \ - --header 'content-type: application/json' \ - --header "Authorization: Bearer $SECRET_CIRRUS_API_KEY" \ - --url 'https://api.cirrus-ci.com/graphql' \ - --data "$query") && [[ -n "$output" ]]; then - - if filtered=$(jq -e "$filter" <<<"$output") && [[ -n "$filtered" ]]; then - msg "result:" - # Make debugging easier w/ formatted output - # to stderr for display, stdout for consumption by caller - jq --indent 2 . <<<"$output" | tee /dev/stderr - msg "::endgroup::" - return 0 - fi - - msg "::error file=${BASH_SOURCE[1]},line=${BASH_LINENO[0]}::Query result did not pass filter '$2': '$output'" - msg "::endgroup::" - return 2 - fi - - msg "::error file=${BASH_SOURCE[1]},line=${BASH_LINENO[0]}::Query failed or result empty: '$output'" - msg "::endgroup::" - return 3 -} diff --git a/.github/actions/check_cirrus_cron/make_email_body.sh b/.github/actions/check_cirrus_cron/make_email_body.sh deleted file mode 100755 index ef013a85ba..0000000000 --- a/.github/actions/check_cirrus_cron/make_email_body.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash - -set -eo pipefail - -# Intended to be executed from a github action workflow step. -# Input: File listing space separated failed cron build names and IDs -# Output: $GITHUB_WORKSPACE/artifacts/email_body.txt file - -source $(dirname "${BASH_SOURCE[0]}")/lib.sh - -_errfmt="Expecting %s value to not be empty" -# ID_NAME_FILEPATH is defined by workflow YAML -# shellcheck disable=SC2154 -if [[ -z "$GITHUB_REPOSITORY" ]]; then - err $(printf "$_errfmt" "\$GITHUB_REPOSITORY") -elif [[ ! -r "$ID_NAME_FILEPATH" ]]; then - err "Expecting \$ID_NAME_FILEPATH value ($ID_NAME_FILEPATH) to be a readable file" -fi - -confirm_gha_environment - -# GITHUB_WORKSPACE confirmed by confirm_gha_environment() -# shellcheck disable=SC2154 -mkdir -p "$GITHUB_WORKSPACE/artifacts" -( - echo "Detected one or more Cirrus-CI cron-triggered jobs have failed recently:" - echo "" - - while read -r BID NAME; do - echo "Cron build '$NAME' Failed: https://cirrus-ci.com/build/$BID" - done < "$ID_NAME_FILEPATH" - - echo "" - # Defined by github-actions - # shellcheck disable=SC2154 - echo "# Source: ${GITHUB_WORKFLOW} workflow on ${GITHUB_REPOSITORY}." - # Separate content from sendgrid.com automatic footer. - echo "" - echo "" -) > $GITHUB_WORKSPACE/artifacts/email_body.txt diff --git a/.github/actions/check_cirrus_cron/rerun_failed_tasks.sh b/.github/actions/check_cirrus_cron/rerun_failed_tasks.sh deleted file mode 100755 index 3c422b066c..0000000000 --- a/.github/actions/check_cirrus_cron/rerun_failed_tasks.sh +++ /dev/null @@ -1,124 +0,0 @@ -#!/bin/bash - -set -eo pipefail - -# Intended to be executed from a github action workflow step. -# Input: File listing space separated failed cron build names and IDs -# Output: $GITHUB_WORKSPACE/artifacts/email_body.txt file -# -# HOW TO TEST: This script may be manually tested assuming you have -# access to the github containers-org. Cirrus API key. With that in-hand, -# this script may be manually run by: -# 1. export SECRET_CIRRUS_API_KEY= -# 2. Find an old podman build that failed on `main` or another **branch**. -# For example, from https://cirrus-ci.com/github/containers/podman/main -# (pick an old one from the bottom, since re-running it won't affect anybody) -# 3. Create a temp. file, like /tmp/fail with a single line, of the form: -# -# 4. export ID_NAME_FILEPATH=/tmp/fail -# 5. execute this script, and refresh the build in the WebUI, all unsuccessful -# tasks should change status to running or scheduled. Note: some later -# tasks may remain red as they wait for dependencies to run and pass. -# 6. After each run, cleanup with 'rm -rf $GITHUB_WORKSPACE/artifacts' -# (unless you want to examine them) - -source $(dirname "${BASH_SOURCE[0]}")/lib.sh - -_errfmt="Expecting %s value to not be empty" -# ID_NAME_FILEPATH is defined by workflow YAML -# shellcheck disable=SC2154 -if [[ -z "$SECRET_CIRRUS_API_KEY" ]]; then - err $(printf "$_errfmt" "\$SECRET_CIRRUS_API_KEY") -elif [[ ! -r "$ID_NAME_FILEPATH" ]]; then # output from cron_failures.sh - err $(printf "Expecting %s value to be a readable file" "\$ID_NAME_FILEPATH") -fi - -confirm_gha_environment - -# GITHUB_WORKSPACE confirmed by confirm_gha_environment() -# shellcheck disable=SC2154 -mkdir -p $GITHUB_WORKSPACE/artifacts -# If there are no tasks, don't fail reading the file -truncate -s 0 $GITHUB_WORKSPACE/artifacts/rerun_tids.txt - -cat "$ID_NAME_FILEPATH" | \ - while read -r BID NAME; do - if [[ -z "$NAME" ]]; then - err $(printf "$_errfmt" "\$NAME") - elif [[ -z "$BID" ]]; then - err $(printf "$_errfmt" "\$BID") - fi - - id_status_q=" - query { - build(id: \"$BID\") { - tasks { - id, - status - } - } - } - " - task_id_status=$(gql "$id_status_q" '.data.build.tasks[0]') - # Expected query result like: - # { - # "data": { - # "build": { - # "tasks": [ - # { - # "id": "6321184690667520", - # "status": "COMPLETED" - # }, - # ... - msg "::group::Selecting failed/aborted tasks to re-run" - jq -r -e '.data.build.tasks[] | join(" ")' <<<"$task_id_status" | \ - while read -r TID STATUS; do - if [[ -z "$TID" ]] || [[ -z "$STATUS" ]]; then - # assume empty line and/or end of file - msg "Skipping TID '$TID' with status '$STATUS'" - continue - # Failed task dependencies will have 'aborted' status - elif [[ "$STATUS" == "FAILED" ]] || [[ "$STATUS" == "ABORTED" ]]; then - msg "Rerunning build $BID task $TID" - # Must send result through a file into rerun_tasks array - # because this section is executing in a child-shell - echo "$TID" >> $GITHUB_WORKSPACE/artifacts/rerun_tids.txt - fi - done - declare -a rerun_tasks - mapfile rerun_tasks <$GITHUB_WORKSPACE/artifacts/rerun_tids.txt - msg "::endgroup::" - - if [[ "${#rerun_tasks[*]}" -eq 0 ]]; then - msg "No tasks to re-run for build $BID" - continue; - fi - - msg "::warning::Rerunning ${#rerun_tasks[*]} tasks for build $BID" - # Check-value returned if the gql call was successful - canary=$(uuidgen) - # Ensure the trailing ',' is stripped from the end (would be invalid JSON) - # Rely on shell word-splitting in this case. - # shellcheck disable=SC2048 - task_ids=$(printf '[%s]' $(printf '"%s",' ${rerun_tasks[*]} | head -c -1)) - rerun_m=" - mutation { - batchReRun(input: { - clientMutationId: \"$canary\", - taskIds: $task_ids - } - ) { - clientMutationId - } - } - " - filter='.data.batchReRun.clientMutationId' - if [[ ! "$NAME" =~ "testing" ]]; then # see test.sh - result=$(gql "$rerun_m" "$filter") - if [[ $(jq -r -e "$filter"<<<"$result") != "$canary" ]]; then - err "Attempt to re-run tasks for build $BID failed: ${rerun_tasks[*]}" - fi - else - warn "Test-mode: Would have sent GraphQL request: '$rerun_m'" - fi - done diff --git a/.github/actions/check_cirrus_cron/test.sh b/.github/actions/check_cirrus_cron/test.sh deleted file mode 100644 index 19f2e35287..0000000000 --- a/.github/actions/check_cirrus_cron/test.sh +++ /dev/null @@ -1,101 +0,0 @@ - - -# This script attempts to confirm functional github action scripts. -# It expects to be called from Cirrus-CI, in a special execution -# environment. Any use outside this environment will probably fail. - -set -eo pipefail - -# Defined by setup_environment.sh -# shellcheck disable=SC2154 -if ! ((PREBUILD)); then - echo "Not operating under expected environment" - exit 1 -fi - -expect_regex() { - local expected_regex - local input_file - expected_regex="$1" - input_file="$2" - grep -E -q "$expected_regex" $input_file || \ - die "No match to '$expected_regex' in '$(<$input_file)'" -} - -req_env_vars CIRRUS_CI CIRRUS_REPO_FULL_NAME CIRRUS_WORKING_DIR CIRRUS_BUILD_ID - -# Defined by the CI system -# shellcheck disable=SC2154 -cd $CIRRUS_WORKING_DIR || fail - -header="Testing cirrus-cron github-action script:" -msg "$header cron_failures.sh" - -base=$CIRRUS_WORKING_DIR/.github/actions/check_cirrus_cron -# Don't care about mktemp return value -# shellcheck disable=SC2155 -export GITHUB_OUTPUT=$(mktemp -p '' cron_failures_output_XXXX) -# CIRRUS_REPO_FULL_NAME checked above in req_env_vars -# shellcheck disable=SC2154 -export GITHUB_REPOSITORY="$CIRRUS_REPO_FULL_NAME" -# shellcheck disable=SC2155 -export GITHUB_WORKSPACE=$(mktemp -d -p '' cron_failures_workspace_XXXX) -export GITHUB_WORKFLOW="testing" -# shellcheck disable=SC2155 -export ID_NAME_FILEPATH=$(mktemp -p '' cron_failures_data_XXXX) -trap "rm -rf $GITHUB_OUTPUT $GITHUB_WORKSPACE $ID_NAME_FILEPATH" EXIT - -##### - -cd $GITHUB_WORKSPACE || fail -# Replace newlines and indentation to make grep easier -if ! $base/cron_failures.sh |& \ - tr -s '[:space:]' ' ' > $GITHUB_WORKSPACE/output; then - die "Failed: $base/cron_failures.sh with output '$(<$GITHUB_WORKSPACE/output)'" -fi - -expect_regex \ - 'result.+data.+ownerRepository.+cronSettings.+endgroup' \ - "$GITHUB_WORKSPACE/output" - -##### - -msg "$header make_email_body.sh" -# It's possible no cirrus-cron jobs actually failed -echo -e '\n\n \n\t\n' >> "$ID_NAME_FILEPATH" # blank lines should be ignored -# Don't need to test stdout/stderr of this -if ! $base/make_email_body.sh; then - die "make_email_body.sh failed" -fi - -expect_regex \ - '^Detected.+Cirrus-CI.+failed.*' \ - "$GITHUB_WORKSPACE/artifacts/email_body.txt" - -##### - -msg "$header make_email_body.sh name and link" -# Job names may contain spaces, confirm lines are parsed properly -echo -e '1234567890 cirrus-cron test job' >> "$ID_NAME_FILEPATH" # Append to blank lines -$base/make_email_body.sh -expected="Cron build 'cirrus-cron test job' Failed: https://cirrus-ci.com/build/1234567890" -if ! grep -q "$expected" $GITHUB_WORKSPACE/artifacts/email_body.txt; then - die "Expecting to find string '$expected' in generated e-mail body: -$(<$GITHUB_WORKSPACE/artifacts/email_body.txt)" -fi - -##### - -msg "$header rerun_failed_tasks.sh" -export SECRET_CIRRUS_API_KEY=testing-nottherightkey -# test.sh is sensitive to the 'testing' name. Var. defined by cirrus-ci -# shellcheck disable=SC2154 -echo "$CIRRUS_BUILD_ID test cron job name" > "$ID_NAME_FILEPATH" -if ! $base/rerun_failed_tasks.sh |& \ - tr -s '[:space:]' ' ' > $GITHUB_WORKSPACE/rerun_output; then - die "rerun_failed_tasks.sh failed" -fi - -expect_regex \ - "Posting GraphQL Query.+$CIRRUS_BUILD_ID.+Selecting.+re-run" \ - "$GITHUB_WORKSPACE/rerun_output"