improve: harden startup report run-id workflows

Co-authored-by: Richie McIlroy <richiemcilroy@users.noreply.github.com>

Author: cursoragent

crates/editor/PLAYBACK-BENCHMARKS.md

@@ -117,6 +117,9 @@ cargo run -p cap-editor --example playback-startup-report -- --log /path/to/edit
 # Filter startup CSV events to a specific labeled run id
 cargo run -p cap-editor --example playback-startup-report -- --log /tmp/playback-startup.csv --run-id macos-pass-1
 
+# List run-id sample counts discovered in startup CSV logs
+cargo run -p cap-editor --example playback-startup-report -- --log /tmp/playback-startup.csv --list-runs
+
 # Aggregate multiple session logs
 cargo run -p cap-editor --example playback-startup-report -- --log /path/to/macos.log --log /path/to/windows.log
 
@@ -178,6 +181,19 @@ cargo run -p cap-recording --example playback-test-runner -- full
   - `cargo run -p cap-editor --example playback-startup-report -- --log crates/editor/PLAYBACK-BENCHMARKS.md --run-id sample-run`
   - Completed successfully with filtered metric output path active.
 
+### Benchmark Run: 2026-02-14 00:00:00 UTC (startup report run-id listing + strict filtering)
+
+**Environment:** Linux runner, startup report parser validation  
+**Commands:** `playback-startup-report --list-runs`, `playback-startup-report --run-id ...`
+
+#### Startup Report CLI Validation
+- `--list-runs` mode prints grouped run-id sample counts from CSV traces.
+- Requesting a `--run-id` with zero matched startup samples now exits with an explicit failure.
+- Validation commands:
+  - `cargo run -p cap-editor --example playback-startup-report -- --log crates/editor/PLAYBACK-BENCHMARKS.md --list-runs`
+  - `cargo run -p cap-editor --example playback-startup-report -- --log crates/editor/PLAYBACK-BENCHMARKS.md --run-id missing-run` (expected non-zero exit)
+- Unit tests remain green: `cargo test -p cap-editor --example playback-startup-report` (**6 passed**).
+
 ### Benchmark Run: 2026-02-14 00:00:00 UTC (supersession span retune to 20)
 
 **Environment:** Linux runner with synthetic 4k60 and 1080p60 MP4 assets  

crates/editor/PLAYBACK-FINDINGS.md

@@ -1009,7 +1009,9 @@ The CPU RGBA→NV12 conversion was taking 15-25ms per frame for 3024x1964 resolu
 **What was done**:
 1. Extended startup report parser to read optional CSV run-id column.
 2. Added run-id filters for aggregate mode and baseline/candidate comparison mode.
-3. Added parser tests that validate run-id filtering behavior on mixed-run CSV traces.
+3. Added `--list-runs` mode to enumerate run-id sample counts from startup CSV traces.
+4. Added strict failures when a requested run-id filter matches zero startup samples.
+5. Added parser tests that validate run-id filtering behavior on mixed-run CSV traces.
 
 **Changes Made**:
 - `crates/editor/examples/playback-startup-report.rs`
@@ -1018,18 +1020,22 @@ The CPU RGBA→NV12 conversion was taking 15-25ms per frame for 3024x1964 resolu
     - `--run-id`
     - `--baseline-run-id`
     - `--candidate-run-id`
+    - `--list-runs`
   - run-id filter now excludes non-matching CSV rows before metric aggregation
+  - run-id filtered queries now return explicit non-zero exit on zero matches
   - added unit test coverage for run-id-filtered parsing
 - `crates/editor/PLAYBACK-BENCHMARKS.md`
-  - added command examples for run-id filtering and same-file baseline/candidate comparisons
+  - added command examples for run-id filtering, run listing, and same-file baseline/candidate comparisons
 
 **Verification**:
 - `cargo +1.88.0 test -p cap-editor --example playback-startup-report`
-- `cargo +1.88.0 run -p cap-editor --example playback-startup-report -- --log /workspace/crates/editor/PLAYBACK-BENCHMARKS.md --run-id sample-run`
+- `cargo +1.88.0 run -p cap-editor --example playback-startup-report -- --log /workspace/crates/editor/PLAYBACK-BENCHMARKS.md --list-runs`
+- `cargo +1.88.0 run -p cap-editor --example playback-startup-report -- --log /workspace/crates/editor/PLAYBACK-BENCHMARKS.md --run-id missing-run` (expected non-zero exit)
 
 **Results**:
 - ✅ Startup parser supports grouped analysis across repeated sessions in one CSV file.
 - ✅ Baseline/candidate deltas can now target specific labeled runs in shared trace files.
+- ✅ Run-id inventory can be listed before comparisons to avoid manual CSV inspection.
 - ✅ All startup report example tests passing (6/6).
 
 **Stopping point**: macOS/Windows startup captures can remain in a single trace file while still enabling precise per-run before/after reporting.

crates/editor/examples/playback-startup-report.rs

@@ -1,4 +1,5 @@
 use std::{
+    collections::BTreeMap,
     fs::File,
     io::{BufRead, BufReader},
     path::PathBuf,
@@ -12,6 +13,15 @@ struct EventStats {
     audio_prerender_startup_ms: Vec<f64>,
 }
 
+impl EventStats {
+    fn total_samples(&self) -> usize {
+        self.decode_startup_ms.len()
+            + self.render_startup_ms.len()
+            + self.audio_stream_startup_ms.len()
+            + self.audio_prerender_startup_ms.len()
+    }
+}
+
 #[derive(Clone, Copy)]
 struct MetricSummary {
     samples: usize,
@@ -121,9 +131,10 @@ fn parse_log(
     path: &PathBuf,
     stats: &mut EventStats,
     run_id_filter: Option<&str>,
-) -> Result<(), String> {
+) -> Result<usize, String> {
     let file = File::open(path).map_err(|error| format!("open {} / {error}", path.display()))?;
     let reader = BufReader::new(file);
+    let mut matched = 0usize;
 
     for line in reader.lines() {
         let line = line.map_err(|error| format!("read {} / {error}", path.display()))?;
@@ -135,10 +146,22 @@ fn parse_log(
                 }
             }
             match event {
-                "first_decoded_frame" => stats.decode_startup_ms.push(startup_ms),
-                "first_rendered_frame" => stats.render_startup_ms.push(startup_ms),
-                "audio_streaming_callback" => stats.audio_stream_startup_ms.push(startup_ms),
-                "audio_prerender_callback" => stats.audio_prerender_startup_ms.push(startup_ms),
+                "first_decoded_frame" => {
+                    stats.decode_startup_ms.push(startup_ms);
+                    matched += 1;
+                }
+                "first_rendered_frame" => {
+                    stats.render_startup_ms.push(startup_ms);
+                    matched += 1;
+                }
+                "audio_streaming_callback" => {
+                    stats.audio_stream_startup_ms.push(startup_ms);
+                    matched += 1;
+                }
+                "audio_prerender_callback" => {
+                    stats.audio_prerender_startup_ms.push(startup_ms);
+                    matched += 1;
+                }
                 _ => {}
             }
             continue;
@@ -154,16 +177,20 @@ fn parse_log(
 
         if line.contains("Playback first decoded frame ready") {
             stats.decode_startup_ms.push(startup_ms);
+            matched += 1;
         } else if line.contains("Playback first frame rendered") {
             stats.render_startup_ms.push(startup_ms);
+            matched += 1;
         } else if line.contains("Audio streaming callback started") {
             stats.audio_stream_startup_ms.push(startup_ms);
+            matched += 1;
         } else if line.contains("Audio pre-rendered callback started") {
             stats.audio_prerender_startup_ms.push(startup_ms);
+            matched += 1;
         }
     }
 
-    Ok(())
+    Ok(matched)
 }
 
 fn parse_csv_startup_event(line: &str) -> Option<(&str, f64, Option<&str>)> {
@@ -178,11 +205,29 @@ fn parse_csv_startup_event(line: &str) -> Option<(&str, f64, Option<&str>)> {
     Some((event, startup_ms, run_id))
 }
 
+fn collect_run_id_counts(path: &PathBuf) -> Result<BTreeMap<String, usize>, String> {
+    let file = File::open(path).map_err(|error| format!("open {} / {error}", path.display()))?;
+    let reader = BufReader::new(file);
+    let mut counts = BTreeMap::<String, usize>::new();
+
+    for line in reader.lines() {
+        let line = line.map_err(|error| format!("read {} / {error}", path.display()))?;
+        if let Some((_, _, run_id)) = parse_csv_startup_event(&line)
+            && let Some(run_id) = run_id
+        {
+            let entry = counts.entry(run_id.to_string()).or_insert(0);
+            *entry += 1;
+        }
+    }
+
+    Ok(counts)
+}
+
 fn main() {
     let args = std::env::args().skip(1).collect::<Vec<_>>();
     if args.is_empty() {
         eprintln!(
-            "Usage: playback-startup-report [--log <path> ...] [--run-id <id>] [--baseline-log <path> ... --candidate-log <path> ...] [--baseline-run-id <id>] [--candidate-run-id <id>]"
+            "Usage: playback-startup-report [--log <path> ...] [--run-id <id>] [--list-runs] [--baseline-log <path> ... --candidate-log <path> ...] [--baseline-run-id <id>] [--candidate-run-id <id>]"
         );
         std::process::exit(1);
     }
@@ -193,6 +238,7 @@ fn main() {
     let mut run_id: Option<String> = None;
     let mut baseline_run_id: Option<String> = None;
     let mut candidate_run_id: Option<String> = None;
+    let mut list_runs = false;
     let mut index = 0usize;
 
     while index < args.len() {
@@ -251,6 +297,11 @@ fn main() {
                 eprintln!("Missing value for --candidate-run-id");
                 std::process::exit(1);
             }
+            "--list-runs" => {
+                list_runs = true;
+                index += 1;
+                continue;
+            }
             _ => {
                 eprintln!("Unknown argument: {}", args[index]);
                 std::process::exit(1);
@@ -268,6 +319,11 @@ fn main() {
         std::process::exit(1);
     }
 
+    if list_runs && (!baseline_logs.is_empty() || !candidate_logs.is_empty()) {
+        eprintln!("--list-runs supports only --log inputs");
+        std::process::exit(1);
+    }
+
     if baseline_logs.is_empty() && baseline_run_id.is_some() {
         eprintln!("--baseline-run-id requires --baseline-log");
         std::process::exit(1);
@@ -279,13 +335,51 @@ fn main() {
     }
 
     if !logs.is_empty() {
+        if list_runs {
+            let mut aggregated = BTreeMap::<String, usize>::new();
+            for log in &logs {
+                match collect_run_id_counts(log) {
+                    Ok(counts) => {
+                        for (run_id_key, count) in counts {
+                            let entry = aggregated.entry(run_id_key).or_insert(0);
+                            *entry += count;
+                        }
+                    }
+                    Err(error) => {
+                        eprintln!("{error}");
+                        std::process::exit(1);
+                    }
+                }
+            }
+
+            println!("Startup trace run-id counts");
+            if aggregated.is_empty() {
+                println!("no run ids found");
+            } else {
+                for (run_id_key, count) in aggregated {
+                    println!("{run_id_key}: {count}");
+                }
+            }
+            return;
+        }
+
         let mut stats = EventStats::default();
+        let mut matched = 0usize;
         for log in &logs {
-            if let Err(error) = parse_log(log, &mut stats, run_id.as_deref()) {
-                eprintln!("{error}");
-                std::process::exit(1);
+            match parse_log(log, &mut stats, run_id.as_deref()) {
+                Ok(count) => {
+                    matched += count;
+                }
+                Err(error) => {
+                    eprintln!("{error}");
+                    std::process::exit(1);
+                }
             }
         }
+        if run_id.is_some() && matched == 0 {
+            eprintln!("No startup samples matched the requested --run-id");
+            std::process::exit(1);
+        }
 
         println!("Playback startup metrics");
         print_metric("first decoded frame", &stats.decode_startup_ms);
@@ -300,20 +394,44 @@ fn main() {
     if !baseline_logs.is_empty() {
         let mut baseline_stats = EventStats::default();
         let baseline_filter = baseline_run_id.as_deref().or(run_id.as_deref());
+        let mut baseline_matched = 0usize;
         for log in &baseline_logs {
-            if let Err(error) = parse_log(log, &mut baseline_stats, baseline_filter) {
-                eprintln!("{error}");
-                std::process::exit(1);
+            match parse_log(log, &mut baseline_stats, baseline_filter) {
+                Ok(count) => {
+                    baseline_matched += count;
+                }
+                Err(error) => {
+                    eprintln!("{error}");
+                    std::process::exit(1);
+                }
             }
         }
+        if baseline_filter.is_some() && baseline_matched == 0 {
+            eprintln!("No baseline startup samples matched the requested run id filter");
+            std::process::exit(1);
+        }
         let mut candidate_stats = EventStats::default();
         let candidate_filter = candidate_run_id.as_deref().or(run_id.as_deref());
+        let mut candidate_matched = 0usize;
         for log in &candidate_logs {
-            if let Err(error) = parse_log(log, &mut candidate_stats, candidate_filter) {
-                eprintln!("{error}");
-                std::process::exit(1);
+            match parse_log(log, &mut candidate_stats, candidate_filter) {
+                Ok(count) => {
+                    candidate_matched += count;
+                }
+                Err(error) => {
+                    eprintln!("{error}");
+                    std::process::exit(1);
+                }
             }
         }
+        if candidate_filter.is_some() && candidate_matched == 0 {
+            eprintln!("No candidate startup samples matched the requested run id filter");
+            std::process::exit(1);
+        }
+        if baseline_stats.total_samples() == 0 || candidate_stats.total_samples() == 0 {
+            eprintln!("No startup samples available for baseline/candidate comparison");
+            std::process::exit(1);
+        }
 
         println!("Startup delta (candidate - baseline)");
         print_delta(