Skip to content

Commit 7b1303e

Browse files
committed
fix(ci): verify report artifacts after producer jobs finish
1 parent dce756e commit 7b1303e

5 files changed

Lines changed: 118 additions & 55 deletions

File tree

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
name: Internal - Assert native report artifact
2+
on:
3+
workflow_call:
4+
inputs:
5+
artifact-id:
6+
description: Native report artifact produced by the scenario.
7+
required: true
8+
type: string
9+
expected-reports:
10+
description: JSON mapping of expected report filenames to their SHA-256 hashes.
11+
required: true
12+
type: string
13+
permissions: {}
14+
jobs:
15+
verify:
16+
runs-on: ubuntu-latest
17+
steps:
18+
# Read artifacts after the producer finishes, as downstream consumers do.
19+
- name: Assert - Download the native report artifact
20+
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
21+
with:
22+
artifact-ids: ${{ inputs.artifact-id }}
23+
path: downloaded-reports
24+
merge-multiple: true
25+
- name: Assert - Only expected reports are uploaded, with unchanged contents
26+
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
27+
env:
28+
EXPECTED_REPORTS: ${{ inputs.expected-reports }}
29+
with:
30+
script: |
31+
const assert = require('node:assert/strict');
32+
const fs = require('node:fs');
33+
const path = require('node:path');
34+
const {createHash} = require('node:crypto');
35+
const expected = JSON.parse(process.env.EXPECTED_REPORTS);
36+
const files = await (await glob.create('downloaded-reports/**/*', {matchDirectories: false})).glob();
37+
assert.deepEqual(files.map(file => path.basename(file)).sort(), Object.keys(expected).sort());
38+
for (const file of files) {
39+
assert.equal(createHash('sha256').update(fs.readFileSync(file)).digest('hex'), expected[path.basename(file)]);
40+
}

‎.github/workflows/__shared-ci.yml‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,12 +37,33 @@ jobs:
3737
permissions:
3838
contents: read
3939
test-action-test:
40+
strategy:
41+
fail-fast: false
42+
matrix:
43+
scenario:
44+
[
45+
reports,
46+
publication-disabled,
47+
preparation,
48+
composer-command,
49+
failing-command,
50+
failing-tests,
51+
no-reports,
52+
]
4053
uses: $/.github/workflows/__test-action-test.yml
54+
with:
55+
scenario: ${{ matrix.scenario }}
4156
permissions:
4257
contents: read
4358
pull-requests: write
4459
test-action-lint:
60+
strategy:
61+
fail-fast: false
62+
matrix:
63+
scenario: [success, failure, no-reports]
4564
uses: $/.github/workflows/__test-action-lint.yml
65+
with:
66+
scenario: ${{ matrix.scenario }}
4667
permissions:
4768
contents: read
4869
pull-requests: write

‎.github/workflows/__test-action-lint.yml‎

Lines changed: 26 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,20 @@
11
name: Internal - Test PHP lint action
22
on:
33
workflow_call:
4+
inputs:
5+
scenario:
6+
description: Integration scenario to run.
7+
required: true
8+
type: string
49
permissions: {}
510
jobs:
611
lint:
712
runs-on: ubuntu-latest
813
permissions:
914
contents: read
10-
strategy:
11-
fail-fast: false
12-
matrix:
13-
scenario: [success, failure, no-reports]
15+
outputs:
16+
report-artifact-id: ${{ steps.lint.outputs.report-artifact-id }}
17+
expected-reports: ${{ steps.assert.outputs.expected-reports }}
1418
steps:
1519
- name: Arrange - Checkout
1620
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -19,7 +23,7 @@ jobs:
1923
- name: Arrange - Configure native PHPStan reports
2024
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
2125
env:
22-
SCENARIO: ${{ matrix.scenario }}
26+
SCENARIO: ${{ inputs.scenario }}
2327
with:
2428
script: |
2529
const fs = require('node:fs/promises');
@@ -42,21 +46,16 @@ jobs:
4246
coverage: none
4347
- name: Act - Lint without comment permission
4448
id: lint
45-
continue-on-error: ${{ matrix.scenario == 'failure' }}
49+
# The assertion below checks the exact expected outcome for every scenario.
50+
continue-on-error: true
4651
uses: $/actions/lint
4752
with:
4853
working-directory: .test-php-lint
49-
- name: Assert - Download the native lint artifact
50-
if: steps.lint.outputs.report-artifact-id != ''
51-
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
52-
with:
53-
artifact-ids: ${{ steps.lint.outputs.report-artifact-id }}
54-
path: downloaded-reports
55-
merge-multiple: true
56-
- name: Assert - Preserve failures and upload only parsed reports
54+
- name: Assert - Preserve failures and identify the native report
55+
id: assert
5756
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
5857
env:
59-
SCENARIO: ${{ matrix.scenario }}
58+
SCENARIO: ${{ inputs.scenario }}
6059
OUTCOME: ${{ steps.lint.outcome }}
6160
ARTIFACT_ID: ${{ steps.lint.outputs.report-artifact-id }}
6261
with:
@@ -69,14 +68,22 @@ jobs:
6968
return;
7069
}
7170
assert.match(process.env.ARTIFACT_ID, /^\d+$/);
72-
const files = await (await glob.create('downloaded-reports/**/*', {matchDirectories: false})).glob();
73-
assert.equal(files.length, 1);
74-
const report = fs.readFileSync(files[0], 'utf8');
75-
assert.equal(report, fs.readFileSync('.test-php-lint/phpstan-checkstyle.xml', 'utf8'));
71+
const {createHash} = require('node:crypto');
72+
const report = fs.readFileSync('.test-php-lint/phpstan-checkstyle.xml', 'utf8');
73+
core.setOutput('expected-reports', {'phpstan-checkstyle.xml': createHash('sha256').update(report).digest('hex')});
7674
assert.match(report, /<checkstyle\b/);
7775
assert.equal(report.includes('<error'), process.env.SCENARIO === 'failure');
7876
77+
verify-reports:
78+
needs: lint
79+
if: needs.lint.outputs.report-artifact-id != ''
80+
uses: $/.github/workflows/__assert-report-artifact.yml
81+
with:
82+
artifact-id: ${{ needs.lint.outputs.report-artifact-id }}
83+
expected-reports: ${{ needs.lint.outputs.expected-reports }}
84+
7985
individual-comment:
86+
if: inputs.scenario == 'success'
8087
runs-on: ubuntu-latest
8188
permissions:
8289
contents: read

‎.github/workflows/__test-action-test.yml‎

Lines changed: 29 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,21 @@
11
name: Internal - Test PHP test action
22
on:
3-
workflow_call: null
3+
workflow_call:
4+
inputs:
5+
scenario:
6+
description: Integration scenario to run.
7+
required: true
8+
type: string
49
permissions: {}
510
jobs:
611
test:
7-
name: Test (${{ matrix.scenario }})
12+
name: Test (${{ inputs.scenario }})
813
runs-on: ubuntu-latest
914
permissions:
1015
contents: read
11-
strategy:
12-
fail-fast: false
13-
matrix:
14-
include:
15-
- scenario: reports
16-
coverage: github
17-
- scenario: publication-disabled
18-
coverage: ""
19-
- scenario: preparation
20-
coverage: ""
21-
- scenario: composer-command
22-
coverage: ""
23-
- scenario: failing-command
24-
coverage: ""
25-
- scenario: failing-tests
26-
coverage: github
27-
- scenario: no-reports
28-
coverage: ""
16+
outputs:
17+
report-artifact-id: ${{ steps.tests.outputs.report-artifact-id }}
18+
expected-reports: ${{ steps.assert.outputs.expected-reports }}
2919
steps:
3020
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
3121
with:
@@ -34,7 +24,7 @@ jobs:
3424
id: fixture
3525
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3
3626
env:
37-
SCENARIO: ${{ matrix.scenario }}
27+
SCENARIO: ${{ inputs.scenario }}
3828
with:
3929
script: |-
4030
const fs = require('node:fs/promises');
@@ -72,26 +62,21 @@ jobs:
7262
composer-directories: '["."]'
7363
- name: Act - Run project tests without comment permission
7464
id: tests
75-
continue-on-error: ${{ startsWith(matrix.scenario, 'failing-') }}
65+
# The assertion below checks the exact expected outcome for every scenario.
66+
continue-on-error: true
7667
uses: $/actions/test
7768
with:
7869
working-directory: ${{ steps.fixture.outputs.directory }}
79-
coverage: ${{ matrix.coverage }}
80-
- name: Assert - Download the native report artifact
81-
if: steps.tests.outputs.report-artifact-id != ''
82-
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
83-
with:
84-
artifact-ids: ${{ steps.tests.outputs.report-artifact-id }}
85-
path: downloaded-reports
86-
merge-multiple: true
70+
coverage: ${{ (inputs.scenario == 'reports' || inputs.scenario == 'failing-tests') && 'github' || '' }}
8771
- name: Assert - Reports and project preparation
72+
id: assert
8873
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3
8974
env:
9075
DIRECTORY: ${{ steps.fixture.outputs.directory }}
9176
OUTCOME: ${{ steps.tests.outcome }}
92-
SCENARIO: ${{ matrix.scenario }}
77+
SCENARIO: ${{ inputs.scenario }}
9378
ARTIFACT_ID: ${{ steps.tests.outputs.report-artifact-id }}
94-
COVERAGE: ${{ matrix.coverage }}
79+
COVERAGE: ${{ (inputs.scenario == 'reports' || inputs.scenario == 'failing-tests') && 'github' || '' }}
9580
with:
9681
script: |-
9782
const assert = require('node:assert/strict');
@@ -105,9 +90,9 @@ jobs:
10590
return;
10691
}
10792
assert.match(process.env.ARTIFACT_ID, /^\d+$/);
108-
const files = await (await glob.create('downloaded-reports/**/*', {matchDirectories: false})).glob();
109-
assert.deepEqual(files.map(file => path.basename(file)).sort(), process.env.COVERAGE === 'github' ? ['cobertura.xml', 'junit.xml'] : ['junit.xml']);
110-
for (const file of files) assert.equal(fs.readFileSync(file, 'utf8'), fs.readFileSync(path.join(directory, path.basename(file)), 'utf8'));
93+
const {createHash} = require('node:crypto');
94+
const reportNames = process.env.COVERAGE === 'github' ? ['cobertura.xml', 'junit.xml'] : ['junit.xml'];
95+
core.setOutput('expected-reports', Object.fromEntries(reportNames.map(name => [name, createHash('sha256').update(fs.readFileSync(path.join(directory, name))).digest('hex')])));
11196
const junit = fs.readFileSync(path.join(directory, 'junit.xml'), 'utf8');
11297
assert.match(junit, /<testcase\b/);
11398
assert.equal(junit.includes('<failure'), scenario === 'failing-tests');
@@ -121,7 +106,16 @@ jobs:
121106
assert.equal(fs.readFileSync(path.join(directory, 'prepared.txt'), 'utf8'), 'ready');
122107
}
123108
109+
verify-reports:
110+
needs: test
111+
if: needs.test.outputs.report-artifact-id != ''
112+
uses: $/.github/workflows/__assert-report-artifact.yml
113+
with:
114+
artifact-id: ${{ needs.test.outputs.report-artifact-id }}
115+
expected-reports: ${{ needs.test.outputs.expected-reports }}
116+
124117
individual-comment:
118+
if: inputs.scenario == 'reports'
125119
runs-on: ubuntu-latest
126120
permissions:
127121
contents: read

‎docs/development/README.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,8 @@ public CI and documentation workflows. The Composer fixture covers automatic and
1212
lockfile and lockfile-free setup, installation hooks, PHPUnit reports, and PHPStan.
1313
Test-action cases cover Composer-owned report generation, coverage publication,
1414
preparation, custom commands, and failure propagation. Dedicated lint and test
15-
workflows retrieve native artifacts from successful and failed commands, check
15+
workflows retrieve native artifacts in dependent jobs after successful or intentionally
16+
failed commands finish. They compare report names and content hashes, check
1617
that hidden project paths work without uploading unrelated files, and verify
1718
empty artifact outputs when no reports are produced. Read-only jobs exercise
1819
the default comment policy; separate jobs verify opt-in comments with collapsed

0 commit comments

Comments
 (0)