From 8ee82c971b672f93159da44446c2636bc1ac8f56 Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Tue, 6 Oct 2026 11:49:02 +0200 Subject: [PATCH 01/12] ci(benchmarks): add manual notify-slo spike job --- .gitlab/benchmarks/macrobenchmarks.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.gitlab/benchmarks/macrobenchmarks.yml b/.gitlab/benchmarks/macrobenchmarks.yml index 642e0f8ca51..5072f8f44c9 100644 --- a/.gitlab/benchmarks/macrobenchmarks.yml +++ b/.gitlab/benchmarks/macrobenchmarks.yml @@ -425,3 +425,22 @@ notify-slo-breaches: when: always variables: CHANNEL: "apm-python-release" + +# Spike: ERE chain from this repo's CI — resolves the on-call schedule by name, +# fetches the on-call engineer's email via pup (printed to the log, never +# messaged), and posts only to the test channel mentioning the spike owner. +# Keys are this repo's own SSM params, so this exercises the exact production +# semantics. Play manually; safe to remove after the notifier rollout. +notify-slo-spike: + stage: notify + needs: [] + when: manual + tags: ["arch:amd64"] + image: "registry.ddbuild.io/images/benchmarking-platform-tools-notifier:0184e511" + variables: + GIT_STRATEGY: none + script: + - export DD_API_KEY="$(aws ssm get-parameter --name "ci.${CI_PROJECT_NAME}.apm_sdk_slo.dd_api_key.prod" --with-decryption --query Parameter.Value --output text)" + - export DD_APP_KEY="$(aws ssm get-parameter --name "ci.${CI_PROJECT_NAME}.apm_sdk_slo.dd_app_key.prod" --with-decryption --query Parameter.Value --output text)" + - export DD_SITE="datadoghq.com" + - notify-slo From 242f4e23fda09427a11b64d8a19db96ca1d84b10 Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Tue, 6 Oct 2026 15:44:31 +0200 Subject: [PATCH 02/12] ci: move notify-slo spike job to the root pipeline --- .gitlab-ci.yml | 19 +++++++++++++++++++ .gitlab/benchmarks/macrobenchmarks.yml | 19 ------------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9c59922b273..6bb377ce925 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -910,3 +910,22 @@ publishing-gate: needs: null # The gate only echoes; without this, stage ordering downloads every prior job's artifacts. dependencies: [] + +# Spike: ERE chain from this repo's CI — resolves the on-call schedule by name, +# fetches the on-call engineer's email via pup (printed to the log, never +# messaged), and posts only to the test channel mentioning the spike owner. +# Keys are this repo's own SSM params, so this exercises the exact production +# semantics. Play manually; safe to remove after the notifier rollout. +notify-slo-spike: + stage: benchmarks + needs: [] + when: manual + tags: ["arch:amd64"] + image: "registry.ddbuild.io/images/benchmarking-platform-tools-notifier:0184e511" + variables: + GIT_STRATEGY: none + script: + - export DD_API_KEY="$(aws ssm get-parameter --name "ci.${CI_PROJECT_NAME}.apm_sdk_slo.dd_api_key.prod" --with-decryption --query Parameter.Value --output text)" + - export DD_APP_KEY="$(aws ssm get-parameter --name "ci.${CI_PROJECT_NAME}.apm_sdk_slo.dd_app_key.prod" --with-decryption --query Parameter.Value --output text)" + - export DD_SITE="datadoghq.com" + - notify-slo diff --git a/.gitlab/benchmarks/macrobenchmarks.yml b/.gitlab/benchmarks/macrobenchmarks.yml index 5072f8f44c9..642e0f8ca51 100644 --- a/.gitlab/benchmarks/macrobenchmarks.yml +++ b/.gitlab/benchmarks/macrobenchmarks.yml @@ -425,22 +425,3 @@ notify-slo-breaches: when: always variables: CHANNEL: "apm-python-release" - -# Spike: ERE chain from this repo's CI — resolves the on-call schedule by name, -# fetches the on-call engineer's email via pup (printed to the log, never -# messaged), and posts only to the test channel mentioning the spike owner. -# Keys are this repo's own SSM params, so this exercises the exact production -# semantics. Play manually; safe to remove after the notifier rollout. -notify-slo-spike: - stage: notify - needs: [] - when: manual - tags: ["arch:amd64"] - image: "registry.ddbuild.io/images/benchmarking-platform-tools-notifier:0184e511" - variables: - GIT_STRATEGY: none - script: - - export DD_API_KEY="$(aws ssm get-parameter --name "ci.${CI_PROJECT_NAME}.apm_sdk_slo.dd_api_key.prod" --with-decryption --query Parameter.Value --output text)" - - export DD_APP_KEY="$(aws ssm get-parameter --name "ci.${CI_PROJECT_NAME}.apm_sdk_slo.dd_app_key.prod" --with-decryption --query Parameter.Value --output text)" - - export DD_SITE="datadoghq.com" - - notify-slo From 411a8e9fd4b8fb7e83bc0e5ff524891b0536a326 Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Tue, 6 Oct 2026 17:30:06 +0200 Subject: [PATCH 03/12] ci: add manual pup write spike job (series + change event) --- .gitlab-ci.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6bb377ce925..818786eb874 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -929,3 +929,32 @@ notify-slo-spike: - export DD_APP_KEY="$(aws ssm get-parameter --name "ci.${CI_PROJECT_NAME}.apm_sdk_slo.dd_app_key.prod" --with-decryption --query Parameter.Value --output text)" - export DD_SITE="datadoghq.com" - notify-slo + +# Spike: validate the write side of pup from this repo's CI — one tagged test +# metric series (v2/series) and one test change event (v2/events), the two write +# calls the notifier redesign depends on. Read-only spikes proved the on-call +# chain; this proves submission. Play manually; safe to remove after rollout. +notify-write-spike: + stage: benchmarks + needs: [] + when: manual + tags: ["arch:amd64"] + image: "registry.ddbuild.io/images/benchmarking-platform-tools-notifier:0184e511" + variables: + GIT_STRATEGY: none + script: + - export DD_API_KEY="$(aws ssm get-parameter --name "ci.${CI_PROJECT_NAME}.apm_sdk_slo.dd_api_key.prod" --with-decryption --query Parameter.Value --output text)" + - export DD_APP_KEY="$(aws ssm get-parameter --name "ci.${CI_PROJECT_NAME}.apm_sdk_slo.dd_app_key.prod" --with-decryption --query Parameter.Value --output text)" + - export DD_SITE="datadoghq.com" + - | + echo "[1/2] Submitting a test metric series via pup (v2/series)..." + now=$(date +%s) + pup --no-agent api -X POST v2/series --input - < Date: Tue, 6 Oct 2026 17:56:14 +0200 Subject: [PATCH 04/12] ci: probe pipeline jobs API with the CI job token in write spike --- .gitlab-ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 818786eb874..94ad904a65e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -957,4 +957,8 @@ notify-write-spike: pup --no-agent api -X POST v2/events --input - < Date: Tue, 6 Oct 2026 18:14:57 +0200 Subject: [PATCH 05/12] ci: add DDCI access spike job --- .gitlab-ci.yml | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 94ad904a65e..285ff36b057 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -962,3 +962,51 @@ notify-write-spike: echo "[3/3] Probing GitLab pipeline jobs API with the CI job token..." curl -sS -w "\nHTTP %{http_code}\n" --header "JOB-TOKEN: $CI_JOB_TOKEN" "$CI_API_V4_URL/projects/$CI_PROJECT_ID/pipelines/$CI_PIPELINE_ID/jobs?per_page=5" | head -c 2000 || true - echo "Done." + +# Spike: validate DDCI access from a CI job — mint a DDCI token via the +# runner's conversion path (id_tokens + DDCI_CONVERT_TO_TICINO_TOKENS), then +# query the DDCI REST API for this request and its tasks. Answers: does the +# job-minted token authenticate to ddci-api, is DDCI context present in this +# job, and do pipeline jobs appear as tasks. Play manually; safe to remove. +ddci-spike: + stage: benchmarks + needs: [] + when: manual + tags: ["arch:amd64"] + image: "registry.ddbuild.io/images/benchmarking-platform-tools-notifier:0184e511" + variables: + GIT_STRATEGY: none + DDCI_CONVERT_TO_TICINO_TOKENS: "DDCI_API_ID_TOKEN" + id_tokens: + DDCI_API_ID_TOKEN: + aud: rapid-devex-ci + script: + - | + echo "[1/3] DDCI context in this job:" + echo "DDCI_REQUEST_ID: ${DDCI_REQUEST_ID:-}" + python3 - <<'PY' + import base64, json, os + tok = os.environ.get("DDCI_API_ID_TOKEN", "") + if not tok: + print("DDCI_API_ID_TOKEN: ") + else: + payload = tok.split(".")[1] + payload += "=" * (-len(payload) % 4) + claims = json.loads(base64.urlsafe_b64decode(payload)) + print("token claims (payload only, never the token):") + print(json.dumps({k: claims.get(k) for k in ("iss", "aud", "name", "gitlab", "code", "ddci")}, indent=1)) + PY + - | + if [ -z "${DDCI_REQUEST_ID:-}" ]; then + echo "[2/3] No DDCI_REQUEST_ID in this job — the DDCI context question answers NEGATIVE for this pipeline type. Not calling the API." + echo "[3/3] skipped." + else + echo "[2/3] GET /requests/ with the job-minted token..." + curl -sS -m 10 --header "Authorization: Bearer $DDCI_API_ID_TOKEN" \ + "https://ddci-api.us1.ddbuild.io/internal/api/v2/requests/${DDCI_REQUEST_ID}" \ + -w "\n--- HTTP %{http_code} ---\n" | head -c 2000 || true + echo "[3/3] GET /requests//tasks with the job-minted token..." + curl -sS -m 10 --header "Authorization: Bearer $DDCI_API_ID_TOKEN" \ + "https://ddci-api.us1.ddbuild.io/internal/api/v2/requests/${DDCI_REQUEST_ID}/tasks" \ + -w "\n--- HTTP %{http_code} ---\n" | head -c 4000 || true + fi From c4b2232a395425fae55b61382904cc9365d6a995 Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Tue, 6 Oct 2026 18:21:29 +0200 Subject: [PATCH 06/12] ci: add BTI PAT spike job --- .gitlab-ci.yml | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 285ff36b057..a058907b977 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1010,3 +1010,37 @@ ddci-spike: "https://ddci-api.us1.ddbuild.io/internal/api/v2/requests/${DDCI_REQUEST_ID}/tasks" \ -w "\n--- HTTP %{http_code} ---\n" | head -c 4000 || true fi + +# Spike: validate the BTI PAT route to the GitLab jobs API from CI — the +# proven pattern from summarize_failures (same repo, same runners) and +# logs-backend. Answers: can this job mint a rapid-devex-ci token via +# authanywhere, exchange it at BTI for a short-lived repo PAT, and list its +# own pipeline's jobs? This is the mechanism the notifier redesign's check +# job will use for benchmark-job outcomes. Play manually; safe to remove. +bti-spike: + stage: benchmarks + needs: [] + when: manual + tags: ["arch:amd64"] + image: "registry.ddbuild.io/images/benchmarking-platform-tools-notifier:0184e511" + variables: + GIT_STRATEGY: none + script: + - | + set -eu + echo "[1/3] Downloading authanywhere (same source as summarize_failures)..." + curl --fail --silent --show-error --location --retry 3 \ + -o /usr/local/bin/authanywhere \ + "https://binaries.ddbuild.io/dd-source/authanywhere/LATEST/authanywhere-linux-amd64" + chmod +x /usr/local/bin/authanywhere + echo "[2/3] Minting rapid-devex-ci token, exchanging at BTI for a short-lived GitLab PAT..." + AUTH="$(authanywhere --audience rapid-devex-ci)" + GL_PAT="$(curl -sS --fail --retry 3 --retry-delay 5 -H "$AUTH" \ + "https://bti-ci-api.us1.ddbuild.io/internal/ci/gitlab/token?owner=${CI_PROJECT_NAMESPACE}&repository=${CI_PROJECT_NAME}" \ + | jq -r .token)" + echo "PAT obtained (length ${#GL_PAT})" + echo "[3/3] Listing this pipeline's jobs with the PAT..." + curl -sS -m 10 --header "PRIVATE-TOKEN: ${GL_PAT}" \ + "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/jobs?per_page=100" \ + | jq -r '.[] | "\(.id)\t\(.name)\t\(.status)"' | head -30 + echo "Done." From 1c81e8eb7c61bd59be8d7ea113643f7b192535f4 Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Tue, 6 Oct 2026 18:30:33 +0200 Subject: [PATCH 07/12] ci: add needs-probe job --- .gitlab-ci.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a058907b977..6527d85a36d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1044,3 +1044,19 @@ bti-spike: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/jobs?per_page=100" \ | jq -r '.[] | "\(.id)\t\(.name)\t\(.status)"' | head -30 echo "Done." + +# Probe: does the GitLab jobs API expose `needs` on a job that declares them? +# Created with the pipeline; never needs to run. Its created-status detail is +# queried via the API with a BTI PAT. +needs-probe: + stage: benchmarks + needs: + - job: bti-spike + optional: true + when: manual + tags: ["arch:amd64"] + image: "registry.ddbuild.io/images/benchmarking-platform-tools-notifier:0184e511" + variables: + GIT_STRATEGY: none + script: + - echo needs-probe From 07e11f996b83914a71c67f64c65f608733d7b253 Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Tue, 6 Oct 2026 18:32:40 +0200 Subject: [PATCH 08/12] ci: remove moot needs-probe job --- .gitlab-ci.yml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6527d85a36d..c86e443bb96 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1045,18 +1045,3 @@ bti-spike: | jq -r '.[] | "\(.id)\t\(.name)\t\(.status)"' | head -30 echo "Done." -# Probe: does the GitLab jobs API expose `needs` on a job that declares them? -# Created with the pipeline; never needs to run. Its created-status detail is -# queried via the API with a BTI PAT. -needs-probe: - stage: benchmarks - needs: - - job: bti-spike - optional: true - when: manual - tags: ["arch:amd64"] - image: "registry.ddbuild.io/images/benchmarking-platform-tools-notifier:0184e511" - variables: - GIT_STRATEGY: none - script: - - echo needs-probe From 0e8b87e6cb6e0baf449d384152aa20c743afdd73 Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Wed, 7 Oct 2026 10:22:47 +0200 Subject: [PATCH 09/12] ci: add render spike posting the new notification texts to the test channel --- .gitlab-ci.yml | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c86e443bb96..f2593c9e81b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1045,3 +1045,49 @@ bti-spike: | jq -r '.[] | "\(.id)\t\(.name)\t\(.status)"' | head -30 echo "Done." + +# Spike: render the new SLO gate notification texts (as produced by +# bp-analyzer's markdown_slack reporter golden fixtures) in the real Slack +# client — validating bullets ('-' markers), emoji spacing and link +# auto-linking before the strings land. Posts ONLY to the test channel. +# Play manually; safe to remove after rollout. +render-spike: + stage: benchmarks + needs: [] + when: manual + tags: ["arch:amd64"] + image: "registry.ddbuild.io/images/benchmarking-platform-tools-notifier:0184e511" + variables: + GIT_STRATEGY: none + script: + - | + CHANNEL="apm-dcs-performance-alerts-test" + postmessage "$CHANNEL" "🔔 render spike (augusto/improve-notifications): the four amended SLO gate notification formats — please check bullets, emoji spacing and links." + + postmessage "$CHANNEL" '🟥 Performance SLOs for dd-trace-java were breached! + - startup:insecure-bank:tracing:GlobalTracer / execution_time = 242.60 ms > 10.00 ms — CI job: https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/jobs/993685721. + Fix the regression, or loosen the SLO — Suggestion: 256.17 ms + - startup:insecure-bank:tracing:Agent / execution_time = 8589.00 ms > 1000.00 ms — CI job: https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/jobs/993685721. + Fix the regression, or loosen the SLO — Suggestion: 9093.51 ms + - startup:insecure-bank:tracing:Agent.start / execution_time = 1027.76 ms > 1000.00 ms — CI job: https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/jobs/993685721. + Fix the regression, or loosen the SLO — Suggestion: 1086.58 ms + 2 more scenarios missing — treated as failed checks — access the CI pipeline https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/pipelines/68475106, investigate failing jobs. + ℹ️ what to do: address the regression, or modify the SLO. More info: https://datadoghq.atlassian.net/wiki/spaces/APMINT/pages/5158175217/Performance+Quality+Gates#What-to-do-in-case-of-warnings-(%F0%9F%9F%A8)-or-breaches-(%F0%9F%9F%A5)%3F + 🔗 CI pipeline: https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/pipelines/68475106 - Commit: https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/commit/3e5c0b48a5 - Dashboard: https://app.datadoghq.com/dashboard/w6a-xc3-kyz' + + postmessage "$CHANNEL" '🟨 Performance SLOs for dd-trace-java were almost breached! + - load:petclinic:appsec:high_load / agg_http_req_duration_p50 = 46.41 ms (warns at 43.20 ms) — CI job: https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/jobs/993685722. + Address the regression, or loosen the SLO — Suggestion: 50.31 ms + ℹ️ what to do: consider addressing the regression now, or modify the SLO — acting now prevents future breaches from blocking releases. More info: https://datadoghq.atlassian.net/wiki/spaces/APMINT/pages/5158175217/Performance+Quality+Gates#What-to-do-in-case-of-warnings-(%F0%9F%9F%A8)-or-breaches-(%F0%9F%9F%A5)%3F + 🔗 CI pipeline: https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/pipelines/68475106 - Commit: https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/commit/3e5c0b48a5 - Dashboard: https://app.datadoghq.com/dashboard/w6a-xc3-kyz' + + postmessage "$CHANNEL" '🚫 Performance SLO check found no results for 3 scenarios in dd-trace-java — treated as failed checks. + - another-missing-scenario + - non-existent-scenario:.* + - startup:insecure-bank:.* + ℹ️ what to do: access the CI pipeline https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/pipelines/68475106, investigate failing jobs. + 🔗 CI pipeline: https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/pipelines/68475106 - Commit: https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/commit/3e5c0b48a5 - Dashboard: https://app.datadoghq.com/dashboard/w6a-xc3-kyz' + + postmessage "$CHANNEL" '🚫 Performance SLO check found no benchmark results for dd-trace-java — benchmark jobs failed, not an SLO breach. + ℹ️ what to do: access the CI pipeline https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/pipelines/68475106, investigate failing jobs. + 🔗 CI pipeline: https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/pipelines/68475106 - Commit: https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/commit/3e5c0b48a5 - Dashboard: https://app.datadoghq.com/dashboard/w6a-xc3-kyz' From 2b8e675fc2f124142f3f8ad36712f1ea701183d8 Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Wed, 7 Oct 2026 10:55:18 +0200 Subject: [PATCH 10/12] ci: re-spike render with corrected bullets, backticks and labeled links --- .gitlab-ci.yml | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f2593c9e81b..7c14251e4a8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1091,3 +1091,49 @@ render-spike: postmessage "$CHANNEL" '🚫 Performance SLO check found no benchmark results for dd-trace-java — benchmark jobs failed, not an SLO breach. ℹ️ what to do: access the CI pipeline https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/pipelines/68475106, investigate failing jobs. 🔗 CI pipeline: https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/pipelines/68475106 - Commit: https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/commit/3e5c0b48a5 - Dashboard: https://app.datadoghq.com/dashboard/w6a-xc3-kyz' + +# Spike: render the new SLO gate notification texts (as produced by +# bp-analyzer's markdown_slack reporter golden fixtures) in the real Slack +# client — validating bullets ('-' markers), emoji spacing and link +# auto-linking before the strings land. Posts ONLY to the test channel. +# Play manually; safe to remove after rollout. +render-spike: + stage: benchmarks + needs: [] + when: manual + tags: ["arch:amd64"] + image: "registry.ddbuild.io/images/benchmarking-platform-tools-notifier:0184e511" + variables: + GIT_STRATEGY: none + script: + - | + CHANNEL="apm-dcs-performance-alerts-test" + postmessage "$CHANNEL" "🔔 render spike v2 (bullets •, backticks, labeled links): the four corrected SLO gate notification formats." + + postmessage "$CHANNEL" '🟥 Performance SLOs for dd-trace-java were breached! + • `startup:insecure-bank:tracing:GlobalTracer` / `execution_time` = 242.60 ms > 10.00 ms — CI job: [993685721](https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/jobs/993685721). + Suggestion: 256.17 ms + • `startup:insecure-bank:tracing:Agent` / `execution_time` = 8589.00 ms > 1000.00 ms — CI job: [993685721](https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/jobs/993685721). + Suggestion: 9093.51 ms + • `startup:insecure-bank:tracing:Agent.start` / `execution_time` = 1027.76 ms > 1000.00 ms — CI job: [993685721](https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/jobs/993685721). + Suggestion: 1086.58 ms + 2 more scenarios missing — treated as failed checks — access the CI pipeline [68475106](https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/pipelines/68475106), investigate failing jobs. + ℹ️ What to do: fix the regression, or loosen the SLOs. More info: https://datadoghq.atlassian.net/wiki/spaces/APMINT/pages/5158175217/Performance+Quality+Gates#What-to-do-in-case-of-warnings-(%F0%9F%9F%A8)-or-breaches-(%F0%9F%9F%A5)%3F + 🔗 CI pipeline: [68475106](https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/pipelines/68475106) - Commit: [3e5c0b48a5](https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/commit/3e5c0b48a5) - Dashboard: [APM SDKs Perf SLOs Dashboard](https://app.datadoghq.com/dashboard/w6a-xc3-kyz)' + + postmessage "$CHANNEL" '🟨 Performance SLOs for dd-trace-java were almost breached! + • `load:petclinic:appsec:high_load` / `agg_http_req_duration_p50` = 46.41 ms (warns at 43.20 ms) — CI job: [993685722](https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/jobs/993685722). + Suggestion: 50.31 ms + ℹ️ What to do: consider addressing the regression now, or loosen the SLOs — acting now prevents future breaches from blocking releases. More info: https://datadoghq.atlassian.net/wiki/spaces/APMINT/pages/5158175217/Performance+Quality+Gates#What-to-do-in-case-of-warnings-(%F0%9F%9F%A8)-or-breaches-(%F0%9F%9F%A5)%3F + 🔗 CI pipeline: [68475106](https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/pipelines/68475106) - Commit: [3e5c0b48a5](https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/commit/3e5c0b48a5) - Dashboard: [APM SDKs Perf SLOs Dashboard](https://app.datadoghq.com/dashboard/w6a-xc3-kyz)' + + postmessage "$CHANNEL" '🚫 Performance SLO check found no results for 3 scenarios in dd-trace-java — treated as failed checks. + • `another-missing-scenario` + • `non-existent-scenario:.*` + • `startup:insecure-bank:.*` + ℹ️ What to do: access the CI pipeline [68475106](https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/pipelines/68475106), investigate failing jobs. + 🔗 CI pipeline: [68475106](https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/pipelines/68475106) - Commit: [3e5c0b48a5](https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/commit/3e5c0b48a5) - Dashboard: [APM SDKs Perf SLOs Dashboard](https://app.datadoghq.com/dashboard/w6a-xc3-kyz)' + + postmessage "$CHANNEL" '🚫 Performance SLO check found no benchmark results for dd-trace-java — benchmark jobs failed, not an SLO breach. + ℹ️ What to do: access the CI pipeline [68475106](https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/pipelines/68475106), investigate failing jobs. + 🔗 CI pipeline: [68475106](https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/pipelines/68475106) - Commit: [3e5c0b48a5](https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/commit/3e5c0b48a5) - Dashboard: [APM SDKs Perf SLOs Dashboard](https://app.datadoghq.com/dashboard/w6a-xc3-kyz)' From c90f4e3e68a21e55dd7c75fdf6e952d91bc096e5 Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Wed, 7 Oct 2026 11:07:29 +0200 Subject: [PATCH 11/12] ci: re-spike render with pipe links and labeled More info --- .gitlab-ci.yml | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7c14251e4a8..63aab23d71b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1137,3 +1137,49 @@ render-spike: postmessage "$CHANNEL" '🚫 Performance SLO check found no benchmark results for dd-trace-java — benchmark jobs failed, not an SLO breach. ℹ️ What to do: access the CI pipeline [68475106](https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/pipelines/68475106), investigate failing jobs. 🔗 CI pipeline: [68475106](https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/pipelines/68475106) - Commit: [3e5c0b48a5](https://gitlab.ddbuild.io/DataDog/dd-trace-java/-/commit/3e5c0b48a5) - Dashboard: [APM SDKs Perf SLOs Dashboard](https://app.datadoghq.com/dashboard/w6a-xc3-kyz)' + +# Spike: render the new SLO gate notification texts (as produced by +# bp-analyzer's markdown_slack reporter golden fixtures) in the real Slack +# client — validating bullets ('-' markers), emoji spacing and link +# auto-linking before the strings land. Posts ONLY to the test channel. +# Play manually; safe to remove after rollout. +render-spike: + stage: benchmarks + needs: [] + when: manual + tags: ["arch:amd64"] + image: "registry.ddbuild.io/images/benchmarking-platform-tools-notifier:0184e511" + variables: + GIT_STRATEGY: none + script: + - | + CHANNEL="apm-dcs-performance-alerts-test" + postmessage "$CHANNEL" "🔔 render spike v3 (pipe links, labeled More info): the four corrected SLO gate notification formats." + + postmessage "$CHANNEL" '🟥 Performance SLOs for dd-trace-java were breached! + • `startup:insecure-bank:tracing:GlobalTracer` / `execution_time` = 242.60 ms > 10.00 ms — CI job: . + Suggestion: 256.17 ms + • `startup:insecure-bank:tracing:Agent` / `execution_time` = 8589.00 ms > 1000.00 ms — CI job: . + Suggestion: 9093.51 ms + • `startup:insecure-bank:tracing:Agent.start` / `execution_time` = 1027.76 ms > 1000.00 ms — CI job: . + Suggestion: 1086.58 ms + 2 more scenarios missing — treated as failed checks — access the CI pipeline , investigate failing jobs. + ℹ️ What to do: fix the regression, or loosen the SLOs. More info: + 🔗 CI pipeline: - Commit: - Dashboard: ' + + postmessage "$CHANNEL" '🟨 Performance SLOs for dd-trace-java were almost breached! + • `load:petclinic:appsec:high_load` / `agg_http_req_duration_p50` = 46.41 ms (warns at 43.20 ms) — CI job: . + Suggestion: 50.31 ms + ℹ️ What to do: consider addressing the regression now, or loosen the SLOs — acting now prevents future breaches from blocking releases. More info: + 🔗 CI pipeline: - Commit: - Dashboard: ' + + postmessage "$CHANNEL" '🚫 Performance SLO check found no results for 3 scenarios in dd-trace-java — treated as failed checks. + • `another-missing-scenario` + • `non-existent-scenario:.*` + • `startup:insecure-bank:.*` + ℹ️ What to do: access the CI pipeline , investigate failing jobs. + 🔗 CI pipeline: - Commit: - Dashboard: ' + + postmessage "$CHANNEL" '🚫 Performance SLO check found no benchmark results for dd-trace-java — benchmark jobs failed, not an SLO breach. + ℹ️ What to do: access the CI pipeline , investigate failing jobs. + 🔗 CI pipeline: - Commit: - Dashboard: ' From 1e628d1fdb1e0b59d8f17370ca9b39c399d88abb Mon Sep 17 00:00:00 2001 From: Augusto de Oliveira Date: Wed, 7 Oct 2026 11:39:33 +0200 Subject: [PATCH 12/12] ci: spike suggestion-line nesting variants --- .gitlab-ci.yml | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 63aab23d71b..806824c98a1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1183,3 +1183,45 @@ render-spike: postmessage "$CHANNEL" '🚫 Performance SLO check found no benchmark results for dd-trace-java — benchmark jobs failed, not an SLO breach. ℹ️ What to do: access the CI pipeline , investigate failing jobs. 🔗 CI pipeline: - Commit: - Dashboard: ' + +# Spike: render the new SLO gate notification texts (as produced by +# bp-analyzer's markdown_slack reporter golden fixtures) in the real Slack +# client — validating bullets ('-' markers), emoji spacing and link +# auto-linking before the strings land. Posts ONLY to the test channel. +# Play manually; safe to remove after rollout. +render-spike: + stage: benchmarks + needs: [] + when: manual + tags: ["arch:amd64"] + image: "registry.ddbuild.io/images/benchmarking-platform-tools-notifier:0184e511" + variables: + GIT_STRATEGY: none + script: + - | + CHANNEL="apm-dcs-performance-alerts-test" + postmessage "$CHANNEL" "🔔 render spike v4 (SLO update suggestion nesting): variant A = nested bullet, variant B = four-space indent. Both below, same breach message." + + postmessage "$CHANNEL" '🧪 Variant A (nested bullet): + 🟥 Performance SLOs for dd-trace-java were breached! + • `startup:insecure-bank:tracing:GlobalTracer` / `execution_time` = 242.60 ms > 10.00 ms — CI job: . + • SLO update suggestion: 256.17 ms + • `startup:insecure-bank:tracing:Agent` / `execution_time` = 8589.00 ms > 1000.00 ms — CI job: . + • SLO update suggestion: 9093.51 ms + • `startup:insecure-bank:tracing:Agent.start` / `execution_time` = 1027.76 ms > 1000.00 ms — CI job: . + • SLO update suggestion: 1086.58 ms + 2 more scenarios missing — treated as failed checks — access the CI pipeline , investigate failing jobs. + ℹ️ What to do: fix the regression, or loosen the SLOs. More info: + 🔗 CI pipeline: - Commit: - Dashboard: ' + + postmessage "$CHANNEL" '🧪 Variant B (four-space indent): + 🟥 Performance SLOs for dd-trace-java were breached! + • `startup:insecure-bank:tracing:GlobalTracer` / `execution_time` = 242.60 ms > 10.00 ms — CI job: . + SLO update suggestion: 256.17 ms + • `startup:insecure-bank:tracing:Agent` / `execution_time` = 8589.00 ms > 1000.00 ms — CI job: . + SLO update suggestion: 9093.51 ms + • `startup:insecure-bank:tracing:Agent.start` / `execution_time` = 1027.76 ms > 1000.00 ms — CI job: . + SLO update suggestion: 1086.58 ms + 2 more scenarios missing — treated as failed checks — access the CI pipeline , investigate failing jobs. + ℹ️ What to do: fix the regression, or loosen the SLOs. More info: + 🔗 CI pipeline: - Commit: - Dashboard: '