diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9c59922b273..806824c98a1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -910,3 +910,318 @@ publishing-gate: needs: null # The gate only echoes; without this, stage ordering downloads every prior job's artifacts. dependencies: [] + +# Spike: ERE chain from this repo's CI — resolves the on-call schedule by name, +# fetches the on-call engineer's email via pup (printed to the log, never +# messaged), and posts only to the test channel mentioning the spike owner. +# Keys are this repo's own SSM params, so this exercises the exact production +# semantics. Play manually; safe to remove after the notifier rollout. +notify-slo-spike: + stage: benchmarks + needs: [] + when: manual + tags: ["arch:amd64"] + image: "registry.ddbuild.io/images/benchmarking-platform-tools-notifier:0184e511" + variables: + GIT_STRATEGY: none + script: + - export DD_API_KEY="$(aws ssm get-parameter --name "ci.${CI_PROJECT_NAME}.apm_sdk_slo.dd_api_key.prod" --with-decryption --query Parameter.Value --output text)" + - export DD_APP_KEY="$(aws ssm get-parameter --name "ci.${CI_PROJECT_NAME}.apm_sdk_slo.dd_app_key.prod" --with-decryption --query Parameter.Value --output text)" + - export DD_SITE="datadoghq.com" + - notify-slo + +# Spike: validate the write side of pup from this repo's CI — one tagged test +# metric series (v2/series) and one test change event (v2/events), the two write +# calls the notifier redesign depends on. Read-only spikes proved the on-call +# chain; this proves submission. Play manually; safe to remove after rollout. +notify-write-spike: + stage: benchmarks + needs: [] + when: manual + tags: ["arch:amd64"] + image: "registry.ddbuild.io/images/benchmarking-platform-tools-notifier:0184e511" + variables: + GIT_STRATEGY: none + script: + - export DD_API_KEY="$(aws ssm get-parameter --name "ci.${CI_PROJECT_NAME}.apm_sdk_slo.dd_api_key.prod" --with-decryption --query Parameter.Value --output text)" + - export DD_APP_KEY="$(aws ssm get-parameter --name "ci.${CI_PROJECT_NAME}.apm_sdk_slo.dd_app_key.prod" --with-decryption --query Parameter.Value --output text)" + - export DD_SITE="datadoghq.com" + - | + echo "[1/2] Submitting a test metric series via pup (v2/series)..." + now=$(date +%s) + pup --no-agent api -X POST v2/series --input - <}" + python3 - <<'PY' + import base64, json, os + tok = os.environ.get("DDCI_API_ID_TOKEN", "") + if not tok: + print("DDCI_API_ID_TOKEN: ") + else: + payload = tok.split(".")[1] + payload += "=" * (-len(payload) % 4) + claims = json.loads(base64.urlsafe_b64decode(payload)) + print("token claims (payload only, never the token):") + print(json.dumps({k: claims.get(k) for k in ("iss", "aud", "name", "gitlab", "code", "ddci")}, indent=1)) + PY + - | + if [ -z "${DDCI_REQUEST_ID:-}" ]; then + echo "[2/3] No DDCI_REQUEST_ID in this job — the DDCI context question answers NEGATIVE for this pipeline type. Not calling the API." + echo "[3/3] skipped." + else + echo "[2/3] GET /requests/ with the job-minted token..." + curl -sS -m 10 --header "Authorization: Bearer $DDCI_API_ID_TOKEN" \ + "https://ddci-api.us1.ddbuild.io/internal/api/v2/requests/${DDCI_REQUEST_ID}" \ + -w "\n--- HTTP %{http_code} ---\n" | head -c 2000 || true + echo "[3/3] GET /requests//tasks with the job-minted token..." + curl -sS -m 10 --header "Authorization: Bearer $DDCI_API_ID_TOKEN" \ + "https://ddci-api.us1.ddbuild.io/internal/api/v2/requests/${DDCI_REQUEST_ID}/tasks" \ + -w "\n--- HTTP %{http_code} ---\n" | head -c 4000 || true + fi + +# Spike: validate the BTI PAT route to the GitLab jobs API from CI — the +# proven pattern from summarize_failures (same repo, same runners) and +# logs-backend. Answers: can this job mint a rapid-devex-ci token via +# authanywhere, exchange it at BTI for a short-lived repo PAT, and list its +# own pipeline's jobs? This is the mechanism the notifier redesign's check +# job will use for benchmark-job outcomes. Play manually; safe to remove. +bti-spike: + stage: benchmarks + needs: [] + when: manual + tags: ["arch:amd64"] + image: "registry.ddbuild.io/images/benchmarking-platform-tools-notifier:0184e511" + variables: + GIT_STRATEGY: none + script: + - | + set -eu + echo "[1/3] Downloading authanywhere (same source as summarize_failures)..." + curl --fail --silent --show-error --location --retry 3 \ + -o /usr/local/bin/authanywhere \ + "https://binaries.ddbuild.io/dd-source/authanywhere/LATEST/authanywhere-linux-amd64" + chmod +x /usr/local/bin/authanywhere + echo "[2/3] Minting rapid-devex-ci token, exchanging at BTI for a short-lived GitLab PAT..." + AUTH="$(authanywhere --audience rapid-devex-ci)" + GL_PAT="$(curl -sS --fail --retry 3 --retry-delay 5 -H "$AUTH" \ + "https://bti-ci-api.us1.ddbuild.io/internal/ci/gitlab/token?owner=${CI_PROJECT_NAMESPACE}&repository=${CI_PROJECT_NAME}" \ + | jq -r .token)" + echo "PAT obtained (length ${#GL_PAT})" + echo "[3/3] Listing this pipeline's jobs with the PAT..." + curl -sS -m 10 --header "PRIVATE-TOKEN: ${GL_PAT}" \ + "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/jobs?per_page=100" \ + | jq -r '.[] | "\(.id)\t\(.name)\t\(.status)"' | head -30 + echo "Done." + + +# Spike: render the new SLO gate notification texts (as produced by +# bp-analyzer's markdown_slack reporter golden fixtures) in the real Slack +# client — validating bullets ('-' markers), emoji spacing and link +# auto-linking before the strings land. Posts ONLY to the test channel. +# Play manually; safe to remove after rollout. +render-spike: + stage: benchmarks + needs: [] + when: manual + tags: ["arch:amd64"] + image: "registry.ddbuild.io/images/benchmarking-platform-tools-notifier:0184e511" + variables: + GIT_STRATEGY: none + script: + - | + CHANNEL="apm-dcs-performance-alerts-test" + postmessage "$CHANNEL" "🔔 render spike (augusto/improve-notifications): the four amended SLO gate notification formats — please check bullets, emoji spacing and links." + + postmessage "$CHANNEL" '🟥 Performance SLOs for dd-trace-java were breached! + - startup:insecure-bank:tracing:GlobalTracer / execution_time = 242.60 ms > 10.00 ms — CI job: https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/jobs/993685721. + Fix the regression, or loosen the SLO — Suggestion: 256.17 ms + - startup:insecure-bank:tracing:Agent / execution_time = 8589.00 ms > 1000.00 ms — CI job: https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/jobs/993685721. + Fix the regression, or loosen the SLO — Suggestion: 9093.51 ms + - startup:insecure-bank:tracing:Agent.start / execution_time = 1027.76 ms > 1000.00 ms — CI job: https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/jobs/993685721. + Fix the regression, or loosen the SLO — Suggestion: 1086.58 ms + 2 more scenarios missing — treated as failed checks — access the CI pipeline https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/pipelines/68475106, investigate failing jobs. + ℹ️ what to do: address the regression, or modify the SLO. More info: https://datadoghq.atlassian.net/wiki/spaces/APMINT/pages/5158175217/Performance+Quality+Gates#What-to-do-in-case-of-warnings-(%F0%9F%9F%A8)-or-breaches-(%F0%9F%9F%A5)%3F + 🔗 CI pipeline: https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/pipelines/68475106 - Commit: https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/commit/3e5c0b48a5 - Dashboard: https://app.datadoghq.com/dashboard/w6a-xc3-kyz' + + postmessage "$CHANNEL" '🟨 Performance SLOs for dd-trace-java were almost breached! + - load:petclinic:appsec:high_load / agg_http_req_duration_p50 = 46.41 ms (warns at 43.20 ms) — CI job: https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/jobs/993685722. + Address the regression, or loosen the SLO — Suggestion: 50.31 ms + ℹ️ what to do: consider addressing the regression now, or modify the SLO — acting now prevents future breaches from blocking releases. More info: https://datadoghq.atlassian.net/wiki/spaces/APMINT/pages/5158175217/Performance+Quality+Gates#What-to-do-in-case-of-warnings-(%F0%9F%9F%A8)-or-breaches-(%F0%9F%9F%A5)%3F + 🔗 CI pipeline: https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/pipelines/68475106 - Commit: https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/commit/3e5c0b48a5 - Dashboard: https://app.datadoghq.com/dashboard/w6a-xc3-kyz' + + postmessage "$CHANNEL" '🚫 Performance SLO check found no results for 3 scenarios in dd-trace-java — treated as failed checks. + - another-missing-scenario + - non-existent-scenario:.* + - startup:insecure-bank:.* + ℹ️ what to do: access the CI pipeline https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/pipelines/68475106, investigate failing jobs. + 🔗 CI pipeline: https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/pipelines/68475106 - Commit: https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/commit/3e5c0b48a5 - Dashboard: https://app.datadoghq.com/dashboard/w6a-xc3-kyz' + + postmessage "$CHANNEL" '🚫 Performance SLO check found no benchmark results for dd-trace-java — benchmark jobs failed, not an SLO breach. + ℹ️ what to do: access the CI pipeline https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/pipelines/68475106, investigate failing jobs. + 🔗 CI pipeline: https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/pipelines/68475106 - Commit: https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/commit/3e5c0b48a5 - Dashboard: https://app.datadoghq.com/dashboard/w6a-xc3-kyz' + +# Spike: render the new SLO gate notification texts (as produced by +# bp-analyzer's markdown_slack reporter golden fixtures) in the real Slack +# client — validating bullets ('-' markers), emoji spacing and link +# auto-linking before the strings land. Posts ONLY to the test channel. +# Play manually; safe to remove after rollout. +render-spike: + stage: benchmarks + needs: [] + when: manual + tags: ["arch:amd64"] + image: "registry.ddbuild.io/images/benchmarking-platform-tools-notifier:0184e511" + variables: + GIT_STRATEGY: none + script: + - | + CHANNEL="apm-dcs-performance-alerts-test" + postmessage "$CHANNEL" "🔔 render spike v2 (bullets •, backticks, labeled links): the four corrected SLO gate notification formats." + + postmessage "$CHANNEL" '🟥 Performance SLOs for dd-trace-java were breached! + • `startup:insecure-bank:tracing:GlobalTracer` / `execution_time` = 242.60 ms > 10.00 ms — CI job: [993685721](https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/jobs/993685721). + Suggestion: 256.17 ms + • `startup:insecure-bank:tracing:Agent` / `execution_time` = 8589.00 ms > 1000.00 ms — CI job: [993685721](https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/jobs/993685721). + Suggestion: 9093.51 ms + • `startup:insecure-bank:tracing:Agent.start` / `execution_time` = 1027.76 ms > 1000.00 ms — CI job: [993685721](https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/jobs/993685721). + Suggestion: 1086.58 ms + 2 more scenarios missing — treated as failed checks — access the CI pipeline [68475106](https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/pipelines/68475106), investigate failing jobs. + ℹ️ What to do: fix the regression, or loosen the SLOs. More info: https://datadoghq.atlassian.net/wiki/spaces/APMINT/pages/5158175217/Performance+Quality+Gates#What-to-do-in-case-of-warnings-(%F0%9F%9F%A8)-or-breaches-(%F0%9F%9F%A5)%3F + 🔗 CI pipeline: [68475106](https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/pipelines/68475106) - Commit: [3e5c0b48a5](https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/commit/3e5c0b48a5) - Dashboard: [APM SDKs Perf SLOs Dashboard](https://app.datadoghq.com/dashboard/w6a-xc3-kyz)' + + postmessage "$CHANNEL" '🟨 Performance SLOs for dd-trace-java were almost breached! + • `load:petclinic:appsec:high_load` / `agg_http_req_duration_p50` = 46.41 ms (warns at 43.20 ms) — CI job: [993685722](https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/jobs/993685722). + Suggestion: 50.31 ms + ℹ️ What to do: consider addressing the regression now, or loosen the SLOs — acting now prevents future breaches from blocking releases. More info: https://datadoghq.atlassian.net/wiki/spaces/APMINT/pages/5158175217/Performance+Quality+Gates#What-to-do-in-case-of-warnings-(%F0%9F%9F%A8)-or-breaches-(%F0%9F%9F%A5)%3F + 🔗 CI pipeline: [68475106](https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/pipelines/68475106) - Commit: [3e5c0b48a5](https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/commit/3e5c0b48a5) - Dashboard: [APM SDKs Perf SLOs Dashboard](https://app.datadoghq.com/dashboard/w6a-xc3-kyz)' + + postmessage "$CHANNEL" '🚫 Performance SLO check found no results for 3 scenarios in dd-trace-java — treated as failed checks. + • `another-missing-scenario` + • `non-existent-scenario:.*` + • `startup:insecure-bank:.*` + ℹ️ What to do: access the CI pipeline [68475106](https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/pipelines/68475106), investigate failing jobs. + 🔗 CI pipeline: [68475106](https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/pipelines/68475106) - Commit: [3e5c0b48a5](https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/commit/3e5c0b48a5) - Dashboard: [APM SDKs Perf SLOs Dashboard](https://app.datadoghq.com/dashboard/w6a-xc3-kyz)' + + postmessage "$CHANNEL" '🚫 Performance SLO check found no benchmark results for dd-trace-java — benchmark jobs failed, not an SLO breach. + ℹ️ What to do: access the CI pipeline [68475106](https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/pipelines/68475106), investigate failing jobs. + 🔗 CI pipeline: [68475106](https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/pipelines/68475106) - Commit: [3e5c0b48a5](https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/commit/3e5c0b48a5) - Dashboard: [APM SDKs Perf SLOs Dashboard](https://app.datadoghq.com/dashboard/w6a-xc3-kyz)' + +# Spike: render the new SLO gate notification texts (as produced by +# bp-analyzer's markdown_slack reporter golden fixtures) in the real Slack +# client — validating bullets ('-' markers), emoji spacing and link +# auto-linking before the strings land. Posts ONLY to the test channel. +# Play manually; safe to remove after rollout. +render-spike: + stage: benchmarks + needs: [] + when: manual + tags: ["arch:amd64"] + image: "registry.ddbuild.io/images/benchmarking-platform-tools-notifier:0184e511" + variables: + GIT_STRATEGY: none + script: + - | + CHANNEL="apm-dcs-performance-alerts-test" + postmessage "$CHANNEL" "🔔 render spike v3 (pipe links, labeled More info): the four corrected SLO gate notification formats." + + postmessage "$CHANNEL" '🟥 Performance SLOs for dd-trace-java were breached! + • `startup:insecure-bank:tracing:GlobalTracer` / `execution_time` = 242.60 ms > 10.00 ms — CI job: . + Suggestion: 256.17 ms + • `startup:insecure-bank:tracing:Agent` / `execution_time` = 8589.00 ms > 1000.00 ms — CI job: . + Suggestion: 9093.51 ms + • `startup:insecure-bank:tracing:Agent.start` / `execution_time` = 1027.76 ms > 1000.00 ms — CI job: . + Suggestion: 1086.58 ms + 2 more scenarios missing — treated as failed checks — access the CI pipeline , investigate failing jobs. + ℹ️ What to do: fix the regression, or loosen the SLOs. More info: + 🔗 CI pipeline: - Commit: - Dashboard: ' + + postmessage "$CHANNEL" '🟨 Performance SLOs for dd-trace-java were almost breached! + • `load:petclinic:appsec:high_load` / `agg_http_req_duration_p50` = 46.41 ms (warns at 43.20 ms) — CI job: . + Suggestion: 50.31 ms + ℹ️ What to do: consider addressing the regression now, or loosen the SLOs — acting now prevents future breaches from blocking releases. More info: + 🔗 CI pipeline: - Commit: - Dashboard: ' + + postmessage "$CHANNEL" '🚫 Performance SLO check found no results for 3 scenarios in dd-trace-java — treated as failed checks. + • `another-missing-scenario` + • `non-existent-scenario:.*` + • `startup:insecure-bank:.*` + ℹ️ What to do: access the CI pipeline , investigate failing jobs. + 🔗 CI pipeline: - Commit: - Dashboard: ' + + postmessage "$CHANNEL" '🚫 Performance SLO check found no benchmark results for dd-trace-java — benchmark jobs failed, not an SLO breach. + ℹ️ What to do: access the CI pipeline , investigate failing jobs. + 🔗 CI pipeline: - Commit: - Dashboard: ' + +# Spike: render the new SLO gate notification texts (as produced by +# bp-analyzer's markdown_slack reporter golden fixtures) in the real Slack +# client — validating bullets ('-' markers), emoji spacing and link +# auto-linking before the strings land. Posts ONLY to the test channel. +# Play manually; safe to remove after rollout. +render-spike: + stage: benchmarks + needs: [] + when: manual + tags: ["arch:amd64"] + image: "registry.ddbuild.io/images/benchmarking-platform-tools-notifier:0184e511" + variables: + GIT_STRATEGY: none + script: + - | + CHANNEL="apm-dcs-performance-alerts-test" + postmessage "$CHANNEL" "🔔 render spike v4 (SLO update suggestion nesting): variant A = nested bullet, variant B = four-space indent. Both below, same breach message." + + postmessage "$CHANNEL" '🧪 Variant A (nested bullet): + 🟥 Performance SLOs for dd-trace-java were breached! + • `startup:insecure-bank:tracing:GlobalTracer` / `execution_time` = 242.60 ms > 10.00 ms — CI job: . + • SLO update suggestion: 256.17 ms + • `startup:insecure-bank:tracing:Agent` / `execution_time` = 8589.00 ms > 1000.00 ms — CI job: . + • SLO update suggestion: 9093.51 ms + • `startup:insecure-bank:tracing:Agent.start` / `execution_time` = 1027.76 ms > 1000.00 ms — CI job: . + • SLO update suggestion: 1086.58 ms + 2 more scenarios missing — treated as failed checks — access the CI pipeline , investigate failing jobs. + ℹ️ What to do: fix the regression, or loosen the SLOs. More info: + 🔗 CI pipeline: - Commit: - Dashboard: ' + + postmessage "$CHANNEL" '🧪 Variant B (four-space indent): + 🟥 Performance SLOs for dd-trace-java were breached! + • `startup:insecure-bank:tracing:GlobalTracer` / `execution_time` = 242.60 ms > 10.00 ms — CI job: . + SLO update suggestion: 256.17 ms + • `startup:insecure-bank:tracing:Agent` / `execution_time` = 8589.00 ms > 1000.00 ms — CI job: . + SLO update suggestion: 9093.51 ms + • `startup:insecure-bank:tracing:Agent.start` / `execution_time` = 1027.76 ms > 1000.00 ms — CI job: . + SLO update suggestion: 1086.58 ms + 2 more scenarios missing — treated as failed checks — access the CI pipeline , investigate failing jobs. + ℹ️ What to do: fix the regression, or loosen the SLOs. More info: + 🔗 CI pipeline: - Commit: - Dashboard: '