Skip to content

Commit 6af9364

Browse files
improve: validate per-machine matrix runs in helper script
Co-authored-by: Richie McIlroy <richiemcilroy@users.noreply.github.com>
1 parent 44d06aa commit 6af9364

4 files changed

Lines changed: 42 additions & 3 deletions

File tree

‎crates/editor/PLAYBACK-BENCHMARKS.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,7 @@ Automated helper for machine runs:
111111

112112
```bash
113113
node scripts/run-playback-benchmark-matrix.js --platform macos-13 --gpu apple-silicon --output-dir /tmp/playback-matrix --fps 60 --input-dir /tmp/cap-real-device-tests
114+
node scripts/run-playback-benchmark-matrix.js --platform windows-11 --gpu nvidia-discrete --output-dir /tmp/playback-matrix-windows-nvidia --fps 60 --require-formats mp4,fragmented --input-dir /tmp/cap-real-device-tests
114115
```
115116

116117
| Platform | GPU Class | MP4 Full | Fragmented Full | MP4 Scrub | Fragmented Scrub | Notes |

‎crates/editor/PLAYBACK-FINDINGS.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,7 @@ cargo run -p cap-recording --example playback-test-runner -- full
182182
10. **Added matrix run helper for platform/GPU benchmark execution (2026-02-13)**
183183
- `scripts/run-playback-benchmark-matrix.js` runs `full` and `scrub` scenarios with tagged notes and JSON output.
184184
- Automatically generates aggregate markdown for each machine run directory.
185+
- Performs per-machine post-run validation for required scenarios and optional format requirements.
185186

186187
11. **Added matrix completeness validator (2026-02-13)**
187188
- `scripts/validate-playback-matrix.js` validates required platform/gpu/scenario cells.

‎crates/editor/PLAYBACK-MATRIX-RUNBOOK.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,13 +43,14 @@ node scripts/run-playback-benchmark-matrix.js \
4343
--gpu "<gpu-label>" \
4444
--output-dir "$OUT_DIR" \
4545
--fps 60 \
46+
--require-formats mp4,fragmented \
4647
--input-dir "$INPUT_DIR"
4748
```
4849

4950
Equivalent shortcut:
5051

5152
```bash
52-
pnpm bench:playback:matrix -- --platform "<platform-label>" --gpu "<gpu-label>" --output-dir "$OUT_DIR" --fps 60 --input-dir "$INPUT_DIR"
53+
pnpm bench:playback:matrix -- --platform "<platform-label>" --gpu "<gpu-label>" --output-dir "$OUT_DIR" --fps 60 --require-formats mp4,fragmented --input-dir "$INPUT_DIR"
5354
```
5455

5556
Examples:

‎scripts/run-playback-benchmark-matrix.js‎

Lines changed: 38 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ function parseArgs(argv) {
1212
fps: 60,
1313
recordingPath: null,
1414
inputDir: null,
15+
validate: true,
16+
requireFormats: [],
1517
};
1618

1719
for (let i = 2; i < argv.length; i++) {
@@ -51,14 +53,26 @@ function parseArgs(argv) {
5153
options.inputDir = argv[++i] ?? null;
5254
continue;
5355
}
56+
if (arg === "--skip-validate") {
57+
options.validate = false;
58+
continue;
59+
}
60+
if (arg === "--require-formats") {
61+
const value = argv[++i] ?? "";
62+
options.requireFormats = value
63+
.split(",")
64+
.map((entry) => entry.trim().toLowerCase())
65+
.filter(Boolean);
66+
continue;
67+
}
5468
throw new Error(`Unknown argument: ${arg}`);
5569
}
5670

5771
return options;
5872
}
5973

6074
function usage() {
61-
console.log(`Usage: node scripts/run-playback-benchmark-matrix.js --platform <name> --gpu <name> --output-dir <dir> [--fps 60] [--recording-path <path>] [--input-dir <path>]
75+
console.log(`Usage: node scripts/run-playback-benchmark-matrix.js --platform <name> --gpu <name> --output-dir <dir> [--fps 60] [--recording-path <path>] [--input-dir <path>] [--require-formats mp4,fragmented]
6276
6377
Runs playback benchmark matrix scenarios and writes JSON outputs.
6478
@@ -70,7 +84,9 @@ Required:
7084
Optional:
7185
--fps FPS for benchmark runs (default: 60)
7286
--recording-path Specific recording path
73-
--input-dir Recording discovery directory`);
87+
--input-dir Recording discovery directory
88+
--require-formats Required formats for local validation (comma-separated)
89+
--skip-validate Skip post-run validation`);
7490
}
7591

7692
function run(command, args) {
@@ -156,6 +172,26 @@ function main() {
156172
aggregatePath,
157173
]);
158174
console.log(`Aggregate markdown: ${aggregatePath}`);
175+
176+
if (options.validate) {
177+
const validateArgs = [
178+
"scripts/validate-playback-matrix.js",
179+
"--input",
180+
options.outputDir,
181+
"--no-default-matrix",
182+
"--require-cell",
183+
`${options.platform}:${options.gpu}:full`,
184+
"--require-cell",
185+
`${options.platform}:${options.gpu}:scrub`,
186+
];
187+
188+
if (options.requireFormats.length > 0) {
189+
validateArgs.push("--require-formats", options.requireFormats.join(","));
190+
}
191+
192+
run("node", validateArgs);
193+
console.log("Matrix run validation passed");
194+
}
159195
}
160196

161197
try {

0 commit comments

Comments
 (0)