From 7ddc2471b5d1408a0eca4c00dbf1592eca637c2f Mon Sep 17 00:00:00 2001 From: Shaun Eccles Date: Mon, 5 Oct 2026 19:15:51 +1100 Subject: [PATCH] fix: isolate PyPI upload sidecars from frozen artifacts --- README.md | 19 +++++++++- docs/native-wheels.md | 2 +- examples/planned-wheels/ci.yml | 7 ++++ tests/test_upload_workflow.py | 67 ++++++++++++++++++++++++++++++++++ 4 files changed, 92 insertions(+), 3 deletions(-) create mode 100644 tests/test_upload_workflow.py diff --git a/README.md b/README.md index ca232e3..dff262d 100644 --- a/README.md +++ b/README.md @@ -21,13 +21,13 @@ Keep your build/test gates, same-run artifact downloads, explicit canonical tag- Target rows select native coverage; tests validate coverage against real planner expectations or the constant pure target using retained public release filename/metadata fixtures. Confirm them against the actual source commit before migration. The audio-hotplug fixture is published 0.1.0: 0.2.0 was unavailable when checked, and its PyPI Trusted Publisher was not yet configured. Prepare its migration PR, but an administrator must register the existing `LedFx/audio-hotplug`, `publish.yml`, `pypi` identity before a real release can succeed. -Aubio's manual TestPyPI job remains a separate unchanged caller lane. This production core accepts only canonical tag pushes and has no TestPyPI or arbitrary upload-URL setting. Preserve samplerate's `!cancelled()` plus explicit successful dependency checks: implicit `success()` can suppress a release when a transitive optional job was intentionally skipped. Keep each project's existing version cross-checks (Meson/vcpkg, SCM tags, CMake or package metadata) before builds. +This production core accepts only canonical tag pushes and has no TestPyPI or arbitrary upload-URL setting. Preserve samplerate's `!cancelled()` plus explicit successful dependency checks: implicit `success()` can suppress a release when a transitive optional job was intentionally skipped. Keep each project's existing version cross-checks (Meson/vcpkg, SCM tags, CMake or package metadata) before builds. Download selectors also remain local: aubio needs both `wheels-*` and `cibw-sdist`; audio uses `python-package-distributions`; noise/samplerate use `cibw-*`; native uses `sender-dist`. Never download a different run's output or regenerate its binaries. Retain output provenance and configure upload paths so colliding filenames cannot overwrite one another unnoticed. ## Action interface -Use `LedFx/release-ci/actions/release@`. Moving version tags are for humans, not consumer pins. Include its released `# vX.Y.Z` comment for Renovate tracking. Upgrade all six consumers to the 0.3 pyproject interface and reviewed released SHA/version together. +Use `LedFx/release-ci/actions/release@`. Moving version tags are for humans, not consumer pins. Include its actual released `# vX.Y.Z` comment for Renovate tracking. A reviewed fix may be pinned before its release; use a descriptive comment instead of claiming a version that does not yet exist. Upgrade every shared plan/release pin within a consumer together. Required inputs: `phase`, `dist`, `snapshot`. Optional `project` defaults to `.` and points to the caller checkout containing the canonical `pyproject.toml`; use `project: release-tools` when source is checked out separately. Project/distribution/asset/OCI paths must stay inside the caller workspace; `snapshot` is an owned path under `runner.temp`. Native projects require the planning job's full `wheel-plan` JSON on every phase. Pure projects omit it. `GH_TOKEN` is supplied through the step environment. Publication runs on hosted Linux with Python 3.11+ and `gh`; Docker/buildx and registry logins are needed only for OCI. No publication runtime dependencies are installed. @@ -71,9 +71,16 @@ Pure package sequence in the caller job (native callers additionally pass the sa project: release-tools dist: dist snapshot: ${{ runner.temp }}/release-snapshot.json +- name: Stage verified distributions for PyPI + if: steps.upload.outputs.pypi_upload == 'true' + working-directory: ${{ github.workspace }} + run: | + mkdir pypi-dist + cp -- dist/* pypi-dist/ - if: steps.upload.outputs.pypi_upload == 'true' uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 with: + packages-dir: pypi-dist/ skip-existing: true # Only after matching remote SHA-256 checks. - id: finalize uses: LedFx/release-ci/actions/release@ @@ -86,6 +93,14 @@ Pure package sequence in the caller job (native callers additionally pass the sa snapshot: ${{ runner.temp }}/release-snapshot.json ``` +Keep the frozen `dist/` unchanged for every shared phase and GitHub attestation. +The PyPA uploader generates `.publish.attestation` sidecars beside its packages. +After a successful `check-upload`, create `pypi-dist/` only when upload is needed +and give that copy to `packages-dir`. Plain `mkdir` refuses an existing staging +directory; keep uploader sidecars there rather than deleting them or weakening +exact filename validation. The relative workspace path is visible inside the +uploader's container. Keep the existing remote hash guard before `skip-existing`. + Always retain the snapshot and `${{ steps.provenance.outputs.bundle-path }}` with the pinned upload-artifact action, including failed finalization. The caller's official PyPI action also supplies PyPI's separate PEP 740 publishing attestation; that does not replace GitHub artifact/OCI provenance verification here. LedFx additionally attests `assets/*`, logs in to its two registries, calls `promote`, attests each returned image digest with `push-to-registry: true`, then calls `finalize`. Pass the same assets/digest paths to every phase and retain all attestation bundles. The core verifies file and OCI attestations against the caller repository/workflow, tested source SHA, tag ref and hosted runner before finalization. Never change the verifier to trust the shared action repository as the artifact's source. diff --git a/docs/native-wheels.md b/docs/native-wheels.md index 10a5c96..a2cc3b6 100644 --- a/docs/native-wheels.md +++ b/docs/native-wheels.md @@ -92,7 +92,7 @@ The fixture tests execute real cibuildwheel enumeration on Linux against disposa | Consumer | Migration detail | | --- | --- | | ledfx-senders | Move its five native rows, retaining `target` artifact labels; keep Rust installation, pinned Linux images and macOS overrides. Replace CI's hard-coded `35`/free-threaded `10` wheel-count assertions with plan coverage, not updated counts. Its separate installed-wheel runtime matrix currently repeats Python/OS coverage: treat it as independent smoke-test coverage and update it deliberately when changing that test contract; the planner guarantees cibuildwheel tests and wheel publication coverage, not that separate matrix. Do not claim that matrix automatically expands. | -| aubio-ledfx | Move its five JSON rows with `triplet`/`macosx_deployment_target` string metadata. Filter `fromJSON(plan.matrix).include` using the existing PR platform/mid-stack logic instead of the old literal JSON; tag releases must use all rows and the full unfiltered plan. Keep vcpkg setup, Windows environment/path handling, release version checks, aggregate gate and separate manual TestPyPI lane. | +| aubio-ledfx | Move its five JSON rows with `triplet`/`macosx_deployment_target` string metadata. Filter `fromJSON(plan.matrix).include` using the existing PR platform/mid-stack logic instead of the old literal JSON; tag releases must use all rows and the full unfiltered plan. Keep vcpkg setup, Windows environment/path handling, release version checks and aggregate gate. | | pyfastnoiselite-ledfx | Move six rows. Preserve armv7l QEMU setup, `test-skip`, the ABI3 backend and its reuse/tests. Replace cibuildwheel action plus `extras: uv` with the frozen group command. Nine ABI3 wheels cover both libc families across three Linux architectures, Windows and two Macs. Keep its sdist-built smoke wheel outside publication output. | | python-samplerate-ledfx | Move five rows with explicit architecture; the same CLI architecture now governs plan/build instead of implicit native defaults. Keep the `build[uv]` frontend, test group, sdist smoke test, and optional gates. Preserve `!cancelled()` and explicit successful required dependencies so intentionally skipped optional jobs do not suppress tag publication. | | LedFx / audio-hotplug | Delete the JSON policy and adopt the same reviewed 0.3 pin and pyproject/context interface, retaining one pure wheel plus sdist. LedFx also merges frozen asset and OCI settings into its pyproject; audio needs no release-specific table. No new planning/build jobs, Rust or cibuildwheel dependency are needed. Application/runtime/frozen/Docker/test matrices do not enter this planner. | diff --git a/examples/planned-wheels/ci.yml b/examples/planned-wheels/ci.yml index 3a0af3e..28fed55 100644 --- a/examples/planned-wheels/ci.yml +++ b/examples/planned-wheels/ci.yml @@ -333,10 +333,17 @@ jobs: dist: dist snapshot: ${{ runner.temp }}/sender-release-snapshot.json wheel-plan: ${{ needs.plan.outputs.wheel-plan }} + - name: Stage verified distributions for PyPI + if: steps.upload.outputs.pypi_upload == 'true' + working-directory: ${{ github.workspace }} + run: | + mkdir pypi-dist + cp -- dist/* pypi-dist/ - name: Publish matching missing Python distributions if: steps.upload.outputs.pypi_upload == 'true' uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 with: + packages-dir: pypi-dist/ skip-existing: true # Both preflights require matching remote SHA-256. - name: Verify provenance and finalize existing GitHub draft env: diff --git a/tests/test_upload_workflow.py b/tests/test_upload_workflow.py new file mode 100644 index 0000000..8de1ce1 --- /dev/null +++ b/tests/test_upload_workflow.py @@ -0,0 +1,67 @@ +"""Uploader sidecars must never contaminate frozen distributions.""" + +import os +import re +import subprocess +import textwrap +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def test_upload_sidecars_leave_frozen_inputs_unchanged(tmp_path: Path) -> None: + workflow = (ROOT / "examples/planned-wheels/ci.yml").read_text() + match = re.search( + r"(?m)^ - name: Stage verified distributions for PyPI\n" + r"(?:(?:^ .*\n)|(?:^\n))*?^ run: \|\n" + r"((?:^ .*\n|^\n)+)", + workflow, + ) + assert match is not None, "The uploader needs a separate verified input copy" + stage = workflow.split(" - name: Stage verified distributions for PyPI\n", 1)[ + 1 + ].split("\n - ", 1)[0] + assert "if: steps.upload.outputs.pypi_upload == 'true'" in stage + assert "working-directory: ${{ github.workspace }}" in stage + assert workflow.index("phase: check-upload") < workflow.index( + "Stage verified distributions for PyPI" + ) + uploader = workflow.split("uses: pypa/gh-action-pypi-publish@", 1)[1].split( + "\n - ", 1 + )[0] + assert "packages-dir: pypi-dist/" in uploader + script = textwrap.dedent(match.group(1)) + original = tmp_path / "dist" + original.mkdir() + frozen = { + "example-1.0-py3-none-any.whl": b"tested wheel", + "example-1.0.tar.gz": b"tested sdist", + } + for name, data in frozen.items(): + (original / name).write_bytes(data) + result = subprocess.run( + ["bash", "-euo", "pipefail", "-c", script], + cwd=tmp_path, + env={**os.environ, "GITHUB_WORKSPACE": str(tmp_path)}, + capture_output=True, + check=False, + ) + assert result.returncode == 0, result.stderr + staging = tmp_path / "pypi-dist" + assert {p.name: p.read_bytes() for p in staging.iterdir()} == frozen + for name in frozen: + (staging / (name + ".publish.attestation")).write_bytes( + b"generated PyPI sidecar" + ) + assert {p.name: p.read_bytes() for p in original.iterdir()} == frozen + before_retry = {p.name: p.read_bytes() for p in staging.iterdir()} + retry = subprocess.run( + ["bash", "-euo", "pipefail", "-c", script], + cwd=tmp_path, + env={**os.environ, "GITHUB_WORKSPACE": str(tmp_path)}, + capture_output=True, + check=False, + ) + assert retry.returncode != 0 + assert {p.name: p.read_bytes() for p in staging.iterdir()} == before_retry + assert {p.name: p.read_bytes() for p in original.iterdir()} == frozen