diff --git a/.github/workflows/notify-docs-failure.yml b/.github/workflows/notify-docs-failure.yml index ec95783de..b54df01b0 100644 --- a/.github/workflows/notify-docs-failure.yml +++ b/.github/workflows/notify-docs-failure.yml @@ -2,9 +2,11 @@ name: notify-docs-failure # The docs deploy from the default branch after a change has already merged, # so a failed build has no PR check to show it: the site quietly stops -# updating. Email when a docs run on the default branch fails after one that -# passed. Later failures in the same streak send nothing, so a broken build -# alerts once instead of on every push. +# updating. Email when a docs run on the default branch fails, once per +# failure streak: after an email goes out, later failures (reruns included) +# send nothing until a docs run passes. An email counts as sent only when the +# send step of this workflow succeeded, so a Resend outage does not mute the +# rest of the streak. # read-only token, no checkout, and it only fires for default-branch runs # jactionlint ignore=dangerous-triggers on: @@ -12,47 +14,113 @@ on: workflows: [Deploy VitePress site to Pages] types: [completed] -concurrency: - group: ${{ github.workflow }}-${{ github.event.workflow_run.id }} - cancel-in-progress: false - permissions: actions: read jobs: notify: - name: notify + # The docs workflow's name is part of the job name, so the check below + # can tell which workflow an earlier email was about. + name: notify (${{ github.event.workflow_run.name }}) if: ${{ github.repository == 'jdx/usage' && github.event.workflow_run.head_branch == github.event.repository.default_branch && contains(fromJSON('["push", "workflow_dispatch", "schedule"]'), github.event.workflow_run.event) && contains(fromJSON('["failure", "timed_out", "startup_failure"]'), github.event.workflow_run.conclusion) }} + # One at a time for each docs workflow, so a run sees whether the one + # before it sent an email. The group is on the job, not the workflow: + # GitHub keeps one pending run per group, and a completion whose job is + # skipped must not replace a failure still waiting to notify. + concurrency: + group: ${{ github.workflow }}-${{ github.event.workflow_run.workflow_id }} + cancel-in-progress: false runs-on: ubuntu-latest timeout-minutes: 5 steps: - - name: Send failure email via Resend + - name: Check whether this failure streak was already emailed + id: check env: GH_TOKEN: ${{ github.token }} - RESEND_API_KEY: ${{ secrets.RESEND_API_KEY }} - NOTIFY_EMAIL: ${{ secrets.NOTIFY_EMAIL }} + HAS_KEY: ${{ secrets.RESEND_API_KEY != '' && secrets.NOTIFY_EMAIL != '' }} WORKFLOW: ${{ github.event.workflow_run.name }} WORKFLOW_ID: ${{ github.event.workflow_run.workflow_id }} - RUN_ID: ${{ github.event.workflow_run.id }} - CONCLUSION: ${{ github.event.workflow_run.conclusion }} BRANCH: ${{ github.event.workflow_run.head_branch }} - SHA: ${{ github.event.workflow_run.head_sha }} - RUN_URL: ${{ github.event.workflow_run.html_url }} + FAILED_AT: ${{ github.event.workflow_run.updated_at }} + RUN_ID: ${{ github.event.workflow_run.id }} + ATTEMPT: ${{ github.event.workflow_run.run_attempt }} run: | set -euo pipefail - if [ -z "${RESEND_API_KEY:-}" ] || [ -z "${NOTIFY_EMAIL:-}" ]; then + if [ "$HAS_KEY" != "true" ]; then echo "::warning::RESEND_API_KEY or NOTIFY_EMAIL is unset; skipping notification for ${WORKFLOW} on ${BRANCH}" + echo "send=false" >> "$GITHUB_OUTPUT" exit 0 fi - # The newest earlier run on this branch that reached a verdict. A - # cancelled or skipped run says nothing about the docs, so it is - # passed over. With no earlier run, this failure is the first. - previous=$(gh api "repos/${GITHUB_REPOSITORY}/actions/workflows/${WORKFLOW_ID}/runs?branch=$(jq -rn --arg b "$BRANCH" '$b|@uri')&status=completed&per_page=50" \ - --jq "[.workflow_runs[] | select(.id < ${RUN_ID} and .event != \"pull_request\" and (.conclusion == \"success\" or .conclusion == \"failure\" or .conclusion == \"timed_out\" or .conclusion == \"startup_failure\"))] | sort_by(.id) | last | .conclusion // \"none\"") - if [ "$previous" != "success" ] && [ "$previous" != "none" ]; then - echo "The previous ${WORKFLOW} run on ${BRANCH} also ended in ${previous}; already reported" + # When the docs last passed. updated_at is when a run's latest + # attempt finished, so a passing rerun of an older run counts from + # when it passed. Pull request runs say nothing about this branch. + branch=$(jq -rn --arg b "$BRANCH" '$b|@uri') + since="" + page=1 + while :; do + body=$(gh api "repos/${GITHUB_REPOSITORY}/actions/workflows/${WORKFLOW_ID}/runs?branch=${branch}&status=success&per_page=100&page=${page}") + since=$(jq -r '[.workflow_runs[] | select(.event != "pull_request") | .updated_at] | max // ""' <<<"$body") + if [ -n "$since" ] || [ "$(jq '.workflow_runs | length' <<<"$body")" -lt 100 ]; then + break + fi + page=$((page + 1)) + done + # That list sees only each run's latest attempt. If this failure is + # a rerun, an earlier attempt of it may have passed, which also + # ended the streak. + for ((n = 1; n < ATTEMPT; n++)); do + passed=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}/attempts/${n}" \ + --jq 'select(.conclusion == "success") | .updated_at') + if [[ "$passed" > "$since" ]]; then + since=$passed + fi + done + # A pass that finished after this failure means the docs already + # recovered, so there is nothing to report. + if [ -n "$since" ] && [[ "$since" > "$FAILED_AT" ]]; then + echo "${WORKFLOW} passed on ${BRANCH} at ${since}, after this failure" + echo "send=false" >> "$GITHUB_OUTPUT" exit 0 fi + # Runs of this workflow since then, oldest first, and earlier + # attempts of this run. If one of them sent an email about this docs + # workflow, this failure belongs to a streak already reported. Runs + # whose job was skipped were not failures of the docs on this branch. + self=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" --jq .workflow_id) + list="repos/${GITHUB_REPOSITORY}/actions/workflows/${self}/runs?status=completed&per_page=100" + if [ -n "$since" ]; then + list="${list}&created=$(jq -rn --arg s ">${since}" '$s|@uri')" + fi + runs=$(gh api --paginate "$list" --jq ".workflow_runs[] | select(.id != ${GITHUB_RUN_ID} and .conclusion != \"skipped\") | .id" | sort -n) + runs="${runs} ${GITHUB_RUN_ID}" + for id in $runs; do + sent=$(gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${id}/jobs?filter=all&per_page=100" | + jq -s --arg job "notify (${WORKFLOW})" \ + '[.[].jobs[] | select(.name == $job) | .steps[]? | select(.name == "Send failure email via Resend" and .conclusion == "success")] | length') + if [ "$sent" != 0 ]; then + echo "Run ${id} already emailed about this ${WORKFLOW} failure streak on ${BRANCH}" + echo "send=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + done + echo "since=${since:-never}" >> "$GITHUB_OUTPUT" + echo "send=true" >> "$GITHUB_OUTPUT" + # Keep this step's name: the check above looks for it to tell whether + # an email went out. + - name: Send failure email via Resend + if: ${{ steps.check.outputs.send == 'true' }} + env: + RESEND_API_KEY: ${{ secrets.RESEND_API_KEY }} + NOTIFY_EMAIL: ${{ secrets.NOTIFY_EMAIL }} + WORKFLOW: ${{ github.event.workflow_run.name }} + RUN_ID: ${{ github.event.workflow_run.id }} + CONCLUSION: ${{ github.event.workflow_run.conclusion }} + BRANCH: ${{ github.event.workflow_run.head_branch }} + SHA: ${{ github.event.workflow_run.head_sha }} + RUN_URL: ${{ github.event.workflow_run.html_url }} + SINCE: ${{ steps.check.outputs.since }} + run: | + set -euo pipefail # Build the payload with jq: --arg keeps a branch or workflow name # from breaking out of the JSON string, and @html keeps it from # becoming markup once the mail client renders the body. @@ -74,11 +142,14 @@ jobs: + "
" ) }') - # A retry can duplicate an alert whose response was lost; two - # alerts beat none. - curl --fail-with-body -sS \ + # The idempotency key makes a retry after a lost response deliver + # one email, not two. It names the docs run and the pass that began + # this failure streak, so a rerun of the run after the docs recovered + # starts a new streak with a new key. + curl --fail-with-body -sS --max-time 20 \ --retry 3 --retry-all-errors --retry-delay 2 --retry-max-time 60 \ -X POST 'https://api.resend.com/emails' \ -H "Authorization: Bearer ${RESEND_API_KEY}" \ + -H "Idempotency-Key: docs-failure/${GITHUB_REPOSITORY}/${RUN_ID}/${SINCE}" \ -H 'Content-Type: application/json' \ -d "$payload"