From 6c81371c78ea5a580a5b1fee9502368317b9499a Mon Sep 17 00:00:00 2001 From: jdx <216188+jdx@users.noreply.github.com> Date: Thu, 12 Nov 2026 17:35:31 +0000 Subject: [PATCH 1/2] ci: bring the docs-failure email workflow up to date Move concurrency to the notify job with a group for each docs workflow, skip failures the docs already recovered from, count a pass from an earlier attempt of a rerun docs run, count an email sent in any attempt, key the Resend request on the docs run alone, paginate the run lists, and time out the Resend call. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/notify-docs-failure.yml | 118 +++++++++++++++++----- 1 file changed, 92 insertions(+), 26 deletions(-) diff --git a/.github/workflows/notify-docs-failure.yml b/.github/workflows/notify-docs-failure.yml index ec95783de..db7bf5a39 100644 --- a/.github/workflows/notify-docs-failure.yml +++ b/.github/workflows/notify-docs-failure.yml @@ -2,9 +2,11 @@ name: notify-docs-failure # The docs deploy from the default branch after a change has already merged, # so a failed build has no PR check to show it: the site quietly stops -# updating. Email when a docs run on the default branch fails after one that -# passed. Later failures in the same streak send nothing, so a broken build -# alerts once instead of on every push. +# updating. Email when a docs run on the default branch fails, once per +# failure streak: after an email goes out, later failures (reruns included) +# send nothing until a docs run passes. An email counts as sent only when the +# send step of this workflow succeeded, so a Resend outage does not mute the +# rest of the streak. # read-only token, no checkout, and it only fires for default-branch runs # jactionlint ignore=dangerous-triggers on: @@ -12,47 +14,110 @@ on: workflows: [Deploy VitePress site to Pages] types: [completed] -concurrency: - group: ${{ github.workflow }}-${{ github.event.workflow_run.id }} - cancel-in-progress: false - permissions: actions: read jobs: notify: - name: notify + # The docs workflow's name is part of the job name, so the check below + # can tell which workflow an earlier email was about. + name: notify (${{ github.event.workflow_run.name }}) if: ${{ github.repository == 'jdx/usage' && github.event.workflow_run.head_branch == github.event.repository.default_branch && contains(fromJSON('["push", "workflow_dispatch", "schedule"]'), github.event.workflow_run.event) && contains(fromJSON('["failure", "timed_out", "startup_failure"]'), github.event.workflow_run.conclusion) }} + # One at a time for each docs workflow, so a run sees whether the one + # before it sent an email. The group is on the job, not the workflow: + # GitHub keeps one pending run per group, and a completion whose job is + # skipped must not replace a failure still waiting to notify. + concurrency: + group: ${{ github.workflow }}-${{ github.event.workflow_run.workflow_id }} + cancel-in-progress: false runs-on: ubuntu-latest timeout-minutes: 5 steps: - - name: Send failure email via Resend + - name: Check whether this failure streak was already emailed + id: check env: GH_TOKEN: ${{ github.token }} - RESEND_API_KEY: ${{ secrets.RESEND_API_KEY }} - NOTIFY_EMAIL: ${{ secrets.NOTIFY_EMAIL }} + HAS_KEY: ${{ secrets.RESEND_API_KEY != '' && secrets.NOTIFY_EMAIL != '' }} WORKFLOW: ${{ github.event.workflow_run.name }} WORKFLOW_ID: ${{ github.event.workflow_run.workflow_id }} - RUN_ID: ${{ github.event.workflow_run.id }} - CONCLUSION: ${{ github.event.workflow_run.conclusion }} BRANCH: ${{ github.event.workflow_run.head_branch }} - SHA: ${{ github.event.workflow_run.head_sha }} - RUN_URL: ${{ github.event.workflow_run.html_url }} + FAILED_AT: ${{ github.event.workflow_run.updated_at }} + RUN_ID: ${{ github.event.workflow_run.id }} + ATTEMPT: ${{ github.event.workflow_run.run_attempt }} run: | set -euo pipefail - if [ -z "${RESEND_API_KEY:-}" ] || [ -z "${NOTIFY_EMAIL:-}" ]; then + if [ "$HAS_KEY" != "true" ]; then echo "::warning::RESEND_API_KEY or NOTIFY_EMAIL is unset; skipping notification for ${WORKFLOW} on ${BRANCH}" + echo "send=false" >> "$GITHUB_OUTPUT" exit 0 fi - # The newest earlier run on this branch that reached a verdict. A - # cancelled or skipped run says nothing about the docs, so it is - # passed over. With no earlier run, this failure is the first. - previous=$(gh api "repos/${GITHUB_REPOSITORY}/actions/workflows/${WORKFLOW_ID}/runs?branch=$(jq -rn --arg b "$BRANCH" '$b|@uri')&status=completed&per_page=50" \ - --jq "[.workflow_runs[] | select(.id < ${RUN_ID} and .event != \"pull_request\" and (.conclusion == \"success\" or .conclusion == \"failure\" or .conclusion == \"timed_out\" or .conclusion == \"startup_failure\"))] | sort_by(.id) | last | .conclusion // \"none\"") - if [ "$previous" != "success" ] && [ "$previous" != "none" ]; then - echo "The previous ${WORKFLOW} run on ${BRANCH} also ended in ${previous}; already reported" + # When the docs last passed. updated_at is when a run's latest + # attempt finished, so a passing rerun of an older run counts from + # when it passed. Pull request runs say nothing about this branch. + branch=$(jq -rn --arg b "$BRANCH" '$b|@uri') + since="" + page=1 + while :; do + body=$(gh api "repos/${GITHUB_REPOSITORY}/actions/workflows/${WORKFLOW_ID}/runs?branch=${branch}&status=success&per_page=100&page=${page}") + since=$(jq -r '[.workflow_runs[] | select(.event != "pull_request") | .updated_at] | max // ""' <<<"$body") + if [ -n "$since" ] || [ "$(jq '.workflow_runs | length' <<<"$body")" -lt 100 ]; then + break + fi + page=$((page + 1)) + done + # That list sees only each run's latest attempt. If this failure is + # a rerun, an earlier attempt of it may have passed, which also + # ended the streak. + for ((n = 1; n < ATTEMPT; n++)); do + passed=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}/attempts/${n}" \ + --jq 'select(.conclusion == "success") | .updated_at') + if [[ "$passed" > "$since" ]]; then + since=$passed + fi + done + # A pass that finished after this failure means the docs already + # recovered, so there is nothing to report. + if [ -n "$since" ] && [[ "$since" > "$FAILED_AT" ]]; then + echo "${WORKFLOW} passed on ${BRANCH} at ${since}, after this failure" + echo "send=false" >> "$GITHUB_OUTPUT" exit 0 fi + # Runs of this workflow since then, oldest first. If one of them + # sent an email about this docs workflow, in any attempt, this + # failure belongs to a streak already reported. Runs whose job was + # skipped were not failures of the docs on this branch. + self=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" --jq .workflow_id) + list="repos/${GITHUB_REPOSITORY}/actions/workflows/${self}/runs?status=completed&per_page=100" + if [ -n "$since" ]; then + list="${list}&created=$(jq -rn --arg s ">${since}" '$s|@uri')" + fi + runs=$(gh api --paginate "$list" --jq ".workflow_runs[] | select(.id != ${GITHUB_RUN_ID} and .conclusion != \"skipped\") | .id" | sort -n) + for id in $runs; do + sent=$(gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${id}/jobs?filter=all&per_page=100" | + jq -s --arg job "notify (${WORKFLOW})" \ + '[.[].jobs[] | select(.name == $job) | .steps[]? | select(.name == "Send failure email via Resend" and .conclusion == "success")] | length') + if [ "$sent" != 0 ]; then + echo "Run ${id} already emailed about this ${WORKFLOW} failure streak on ${BRANCH}" + echo "send=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + done + echo "send=true" >> "$GITHUB_OUTPUT" + # Keep this step's name: the check above looks for it to tell whether + # an email went out. + - name: Send failure email via Resend + if: ${{ steps.check.outputs.send == 'true' }} + env: + RESEND_API_KEY: ${{ secrets.RESEND_API_KEY }} + NOTIFY_EMAIL: ${{ secrets.NOTIFY_EMAIL }} + WORKFLOW: ${{ github.event.workflow_run.name }} + RUN_ID: ${{ github.event.workflow_run.id }} + CONCLUSION: ${{ github.event.workflow_run.conclusion }} + BRANCH: ${{ github.event.workflow_run.head_branch }} + SHA: ${{ github.event.workflow_run.head_sha }} + RUN_URL: ${{ github.event.workflow_run.html_url }} + run: | + set -euo pipefail # Build the payload with jq: --arg keeps a branch or workflow name # from breaking out of the JSON string, and @html keeps it from # becoming markup once the mail client renders the body. @@ -74,11 +139,12 @@ jobs: + "

View workflow run

" ) }') - # A retry can duplicate an alert whose response was lost; two - # alerts beat none. - curl --fail-with-body -sS \ + # The idempotency key makes a retry after a lost response deliver + # one email, not two, and covers a rerun of the same docs run too. + curl --fail-with-body -sS --max-time 20 \ --retry 3 --retry-all-errors --retry-delay 2 --retry-max-time 60 \ -X POST 'https://api.resend.com/emails' \ -H "Authorization: Bearer ${RESEND_API_KEY}" \ + -H "Idempotency-Key: docs-failure/${GITHUB_REPOSITORY}/${RUN_ID}" \ -H 'Content-Type: application/json' \ -d "$payload" From dba81d3dd428c0a076cbe6acd9bb6b7f6c239fe1 Mon Sep 17 00:00:00 2001 From: jdx <216188+jdx@users.noreply.github.com> Date: Thu, 12 Nov 2026 17:39:08 +0000 Subject: [PATCH 2/2] ci: key docs-failure emails on the failure streak A docs run that failed and emailed, then failed again on a rerun after another run passed, reused its Resend idempotency key, so Resend treated the new alert as a repeat and sent nothing. The key now also names the pass that began the streak. The check also counts an email sent by an earlier attempt of its own run. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/notify-docs-failure.yml | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/notify-docs-failure.yml b/.github/workflows/notify-docs-failure.yml index db7bf5a39..b54df01b0 100644 --- a/.github/workflows/notify-docs-failure.yml +++ b/.github/workflows/notify-docs-failure.yml @@ -82,16 +82,17 @@ jobs: echo "send=false" >> "$GITHUB_OUTPUT" exit 0 fi - # Runs of this workflow since then, oldest first. If one of them - # sent an email about this docs workflow, in any attempt, this - # failure belongs to a streak already reported. Runs whose job was - # skipped were not failures of the docs on this branch. + # Runs of this workflow since then, oldest first, and earlier + # attempts of this run. If one of them sent an email about this docs + # workflow, this failure belongs to a streak already reported. Runs + # whose job was skipped were not failures of the docs on this branch. self=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" --jq .workflow_id) list="repos/${GITHUB_REPOSITORY}/actions/workflows/${self}/runs?status=completed&per_page=100" if [ -n "$since" ]; then list="${list}&created=$(jq -rn --arg s ">${since}" '$s|@uri')" fi runs=$(gh api --paginate "$list" --jq ".workflow_runs[] | select(.id != ${GITHUB_RUN_ID} and .conclusion != \"skipped\") | .id" | sort -n) + runs="${runs} ${GITHUB_RUN_ID}" for id in $runs; do sent=$(gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${id}/jobs?filter=all&per_page=100" | jq -s --arg job "notify (${WORKFLOW})" \ @@ -102,6 +103,7 @@ jobs: exit 0 fi done + echo "since=${since:-never}" >> "$GITHUB_OUTPUT" echo "send=true" >> "$GITHUB_OUTPUT" # Keep this step's name: the check above looks for it to tell whether # an email went out. @@ -116,6 +118,7 @@ jobs: BRANCH: ${{ github.event.workflow_run.head_branch }} SHA: ${{ github.event.workflow_run.head_sha }} RUN_URL: ${{ github.event.workflow_run.html_url }} + SINCE: ${{ steps.check.outputs.since }} run: | set -euo pipefail # Build the payload with jq: --arg keeps a branch or workflow name @@ -140,11 +143,13 @@ jobs: ) }') # The idempotency key makes a retry after a lost response deliver - # one email, not two, and covers a rerun of the same docs run too. + # one email, not two. It names the docs run and the pass that began + # this failure streak, so a rerun of the run after the docs recovered + # starts a new streak with a new key. curl --fail-with-body -sS --max-time 20 \ --retry 3 --retry-all-errors --retry-delay 2 --retry-max-time 60 \ -X POST 'https://api.resend.com/emails' \ -H "Authorization: Bearer ${RESEND_API_KEY}" \ - -H "Idempotency-Key: docs-failure/${GITHUB_REPOSITORY}/${RUN_ID}" \ + -H "Idempotency-Key: docs-failure/${GITHUB_REPOSITORY}/${RUN_ID}/${SINCE}" \ -H 'Content-Type: application/json' \ -d "$payload"