diff --git a/CHANGELOG.md b/CHANGELOG.md index 4005ef2..565627c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,18 @@ All notable changes to F1 Replay Timing will be documented in this file. +## 2.1.3 + +### Improvements + +- **Storage management** — a **Storage** button in the header shows total disk use and sets a retention policy: sessions older than a chosen number of weeks or months are deleted, immediately and then daily. Individual sessions can be deleted by right-clicking them in the session picker. (requested by [@db-can](https://github.com/db-can)) + +### Fixes + +- **Replay froze after a dropped connection** — an idle replay sent no data, so proxies could close the socket and the client never reconnected. The server now sends a keep-alive every 20s, and the client reconnects and resumes where it left off. ([#106](https://github.com/adn8naiagent/F1ReplayTiming/issues/106), reported by [@nbolland](https://github.com/nbolland)) + +- **App unreachable behind a reverse proxy when `PORT` was set** — 2.1.2 passed `PORT` into the container, moving the port proxies point at (Traefik, nginx and cloudflared all target 8000). The container now always listens on 8000; `PORT` sets the published host port only. (found while investigating [#106](https://github.com/adn8naiagent/F1ReplayTiming/issues/106), reported by [@nbolland](https://github.com/nbolland)) + ## 2.1.2 ### Fixes diff --git a/backend/main.py b/backend/main.py index a5fa4dc..ab3ba51 100644 --- a/backend/main.py +++ b/backend/main.py @@ -13,6 +13,7 @@ from routers import sessions, track, laps, results, replay, telemetry, sync, live, live_status from routers import auth_routes from services.auto_precompute import auto_precompute_loop, get_allowed_session_types +from services.retention import retention_loop load_dotenv() @@ -29,13 +30,19 @@ async def lifespan(app: FastAPI): logger.info(f"Auto-precompute scheduled for session types: {sorted(allowed)}") else: logger.info("Auto-precompute disabled (AUTO_PRECOMPUTE=off)") + # Runs regardless of AUTO_PRECOMPUTE: someone who turned auto-download off + # is exactly who is most likely to be managing disk space. + retention_task = asyncio.create_task(retention_loop()) + yield - if task is not None: - task.cancel() - try: - await task - except asyncio.CancelledError: - pass + + for t in (task, retention_task): + if t is not None: + t.cancel() + try: + await t + except asyncio.CancelledError: + pass app = FastAPI( diff --git a/backend/routers/replay.py b/backend/routers/replay.py index ab9c46e..a0579c6 100644 --- a/backend/routers/replay.py +++ b/backend/routers/replay.py @@ -37,6 +37,15 @@ def _log_memory(): CACHE_EVICTION_SECONDS = 300 # 5 minutes after last client disconnects +# Idle gap after which the server sends a keep-alive frame. Proxies and tunnels +# that only count data frames towards their idle timeout (commonly 60s or 300s) +# will close the socket otherwise — a paused replay sends nothing at all. +HEARTBEAT_SECONDS = 20 + +# Queued by the reader task when the client goes away, so the playback loop can +# raise WebSocketDisconnect from wherever it happens to be waiting. +_DISCONNECTED = object() + # In-memory cache for pit loss data _pit_loss_cache: dict | None = None @@ -206,6 +215,20 @@ async def _client_disconnect(key: str): _eviction_tasks[key] = task +def evict_cached_session(year: int, round_num: int, session_type: str) -> None: + """Drop a session's frames from memory immediately. + + Called when stored data is deleted; without it the frames stay resident + until the normal eviction delay elapses. + """ + key = f"{year}_{round_num}_{session_type}" + task = _eviction_tasks.pop(key, None) + if task: + task.cancel() + if _replay_cache.pop(key, None) is not None: + logger.info(f"[memory] Evicted {key} after delete — {_log_memory()}") + + async def _evict_after_delay(key: str): """Wait, then evict a cached session if no new clients have connected.""" try: @@ -232,6 +255,8 @@ async def replay_websocket( return await websocket.accept() + receiver_task: asyncio.Task | None = None + try: async def send_status(msg: str): await websocket.send_json({"type": "status", "message": msg}) @@ -311,8 +336,20 @@ def prepare_frame(f: dict) -> dict: _add_pit_predictions(f, pit_loss_green, pit_loss_sc, pit_loss_vsc) return f + last_send = time.monotonic() + + async def send_json(payload: dict) -> None: + """Send, recording the time so the heartbeat only fires when idle.""" + nonlocal last_send + await websocket.send_json(payload) + last_send = time.monotonic() + + async def beat_if_idle() -> None: + if time.monotonic() - last_send >= HEARTBEAT_SECONDS: + await send_json({"type": "ping"}) + # Send first frame immediately so cars are visible before play - await websocket.send_json({"type": "frame", **prepare_frame(frames[0])}) + await send_json({"type": "frame", **prepare_frame(frames[0])}) # Playback state playing = False @@ -338,7 +375,7 @@ async def send_seek_frame(target_time: float): frame_index = i break if frame_index < len(frames): - await websocket.send_json({"type": "frame", **prepare_frame(frames[frame_index])}) + await send_json({"type": "frame", **prepare_frame(frames[frame_index])}) async def handle_command(cmd: str): nonlocal playing, speed, frame_index @@ -370,32 +407,48 @@ async def handle_command(cmd: str): frame_index = i break if frame_index < len(frames): - await websocket.send_json({"type": "frame", **prepare_frame(frames[frame_index])}) + await send_json({"type": "frame", **prepare_frame(frames[frame_index])}) reset_anchor() except ValueError: pass elif cmd == "reset": frame_index = 0 playing = False - await websocket.send_json({"type": "frame", **prepare_frame(frames[0])}) + await send_json({"type": "frame", **prepare_frame(frames[0])}) reset_anchor() + # One long-lived reader feeding a queue. The playback loop polls the + # queue on a 50ms tick; cancelling a queue.get() is safe, whereas + # cancelling websocket.receive_text() that often can drop commands. + commands: asyncio.Queue = asyncio.Queue() + + async def receive_commands(): + try: + while True: + await commands.put(await websocket.receive_text()) + except Exception: + await commands.put(_DISCONNECTED) + + receiver_task = asyncio.create_task(receive_commands()) + async def check_command(timeout: float) -> bool: try: - msg = await asyncio.wait_for(websocket.receive_text(), timeout=timeout) - await handle_command(msg.strip().lower()) - return True + msg = await asyncio.wait_for(commands.get(), timeout=timeout) except asyncio.TimeoutError: return False + if msg is _DISCONNECTED: + raise WebSocketDisconnect(1006) + await handle_command(msg.strip().lower()) + return True while True: if playing and frame_index < len(frames): - await websocket.send_json({"type": "frame", **prepare_frame(frames[frame_index])}) + await send_json({"type": "frame", **prepare_frame(frames[frame_index])}) frame_index += 1 if frame_index >= len(frames): playing = False - await websocket.send_json({"type": "finished"}) + await send_json({"type": "finished"}) continue # Sleep until the next frame is due per wall clock. @@ -408,9 +461,13 @@ async def check_command(timeout: float) -> bool: while sleep_remaining > 0 and playing: chunk = min(sleep_remaining, 0.05) await check_command(chunk) + # Covers long gaps between frames: red flags, and slow speeds. + await beat_if_idle() sleep_remaining = target_wall - time.monotonic() else: + # Paused, or waiting to start — nothing else is sent here. await check_command(1.0) + await beat_if_idle() except WebSocketDisconnect: cache_key = f"{year}_{round_num}_{type}" @@ -424,3 +481,6 @@ async def check_command(timeout: float) -> bool: await websocket.close() except Exception: pass + finally: + if receiver_task is not None: + receiver_task.cancel() diff --git a/backend/routers/sessions.py b/backend/routers/sessions.py index 1a7544a..8c0fa3b 100644 --- a/backend/routers/sessions.py +++ b/backend/routers/sessions.py @@ -1,3 +1,4 @@ +import asyncio import logging import time from copy import deepcopy @@ -5,7 +6,21 @@ from fastapi import APIRouter, Query, HTTPException from services.storage import get_json, put_json, list_sizes -from services.process import ensure_session_data, start_reprocess, get_reprocess_status +from services.process import ( + ensure_session_data, + start_reprocess, + get_reprocess_status, + delete_session, +) +from services.retention import ( + RetentionError, + get_retention, + get_sweep_status, + save_retention, + session_index, + start_sweep, + sweep, +) logger = logging.getLogger(__name__) router = APIRouter(prefix="/api", tags=["sessions"]) @@ -192,3 +207,62 @@ async def reprocess_session(year: int, round_num: int, type: str = Query("R")): @router.get("/sessions/{year}/{round_num}/reprocess/status") async def reprocess_session_status(year: int, round_num: int, type: str = Query("R")): return get_reprocess_status(year, round_num, type) + + +@router.get("/storage") +async def storage_usage(): + """Stored size of every downloaded session, plus the retention setting.""" + sessions, total = session_index() + return { + "total_bytes": total, + "session_count": len(sessions), + "sessions": sessions, + "retention": get_retention(), + } + + +@router.delete("/sessions/{year}/{round_num}") +async def delete_session_data(year: int, round_num: int, type: str = Query("R")): + """Delete one session's stored data. Auth is enforced by the /api middleware.""" + if type not in SESSION_NAME_TO_TYPE.values(): + raise HTTPException(status_code=400, detail=f"Unknown session type: {type}") + freed = delete_session(year, round_num, type) + _events_cache.clear() + return {"deleted": 1, "freed_bytes": freed} + + +@router.put("/storage/retention") +async def update_retention( + enabled: bool = Query(...), + amount: int = Query(...), + unit: str = Query("months"), + dry_run: bool = Query(False, description="Report what would be deleted, change nothing"), +): + """Set the retention policy, applying it immediately. + + Call with dry_run=true first to show the user what saving would remove; + saving then deletes everything already older than the threshold, and the + daily sweep keeps it that way. + """ + try: + if dry_run: + return sweep(amount, unit, dry_run=True) + + config = save_retention(enabled, amount, unit) + if not enabled: + return {"retention": config, "started": False} + + # Deleting a large backlog takes a while, so run it in the background + # and let the client poll for progress rather than holding the request. + status = await start_sweep(amount, unit) + except RetentionError as e: + raise HTTPException(status_code=400, detail=str(e)) + + _events_cache.clear() + return {"retention": config, "started": True, "status": status} + + +@router.get("/storage/retention/status") +async def retention_status(): + """Progress of the running or most recent sweep.""" + return get_sweep_status() diff --git a/backend/services/auto_precompute.py b/backend/services/auto_precompute.py index 6746030..cb3fce3 100644 --- a/backend/services/auto_precompute.py +++ b/backend/services/auto_precompute.py @@ -62,7 +62,7 @@ async def _check_and_process(): from services.f1_data import _fetch_schedule_sync, SESSION_NAME_TO_TYPE from services import storage - from services.process import process_session_sync as process_session + from services.process import process_session_sync as process_session, is_deleted now = datetime.now(timezone.utc) year = now.year @@ -108,6 +108,11 @@ async def _check_and_process(): if storage.exists(f"{base}/replay.json"): continue + # Respect an explicit delete — don't silently re-download data the + # user removed. Cleared automatically if they reprocess it. + if is_deleted(year, round_num, session_type): + continue + # New session data might be available - try to process it prefix = f"{year} R{round_num} {session_type}" logger.info(f"[auto] New session detected: {prefix}, attempting precompute...") diff --git a/backend/services/process.py b/backend/services/process.py index 6c00f0a..0ef95f6 100644 --- a/backend/services/process.py +++ b/backend/services/process.py @@ -9,6 +9,7 @@ import asyncio import logging import traceback +from datetime import datetime, timezone from services import storage from services.f1_data import ( @@ -29,6 +30,11 @@ # state is "running" | "done" | "error" _reprocess_status: dict[str, dict] = {} +# Written when a user deletes a session. auto_precompute skips sessions carrying +# one, so an explicit delete isn't undone by the background downloader; any +# successful reprocess clears it. +TOMBSTONE = "deleted.json" + def get_reprocess_status(year: int, round_num: int, session_type: str) -> dict: """Return the current reprocess state + latest progress message for a session.""" @@ -169,6 +175,20 @@ def status(msg: str): except Exception as e: logger.warning(f"[{prefix}] Telemetry upload issue: {e}") + # Session is back — drop any delete tombstone so auto-precompute resumes. + try: + storage.delete(f"{base}/{TOMBSTONE}") + except Exception: + pass + + # Invalidate the cached events listing so the picker shows the downloaded + # marker straight away, rather than after the 5 minute cache expires. + try: + from routers.sessions import _events_cache + _events_cache.clear() + except Exception: + pass + status("Processing complete") logger.info(f"[{prefix}] Done") return True @@ -288,3 +308,39 @@ def sync_status(msg: str): except Exception as e: logger.error(f"On-demand processing failed for {key}: {e}") return False + + +def is_deleted(year: int, round_num: int, session_type: str) -> bool: + """True if the user explicitly deleted this session's data.""" + base = f"sessions/{year}/{round_num}/{session_type}" + return storage.exists(f"{base}/{TOMBSTONE}") + + +def delete_session(year: int, round_num: int, session_type: str) -> int: + """Delete a session's stored data. Returns the number of bytes freed. + + replay.json is removed first, because every availability check in the app + keys off it. If the delete is interrupted the session then reads as "not + downloaded" and reprocesses cleanly, rather than being left present but + incomplete — the failure mode people hit deleting these files by hand. + """ + base = f"sessions/{year}/{round_num}/{session_type}" + + freed = storage.delete(f"{base}/replay.json") + freed += storage.delete_prefix(base) + + storage.put_json( + f"{base}/{TOMBSTONE}", + {"deleted_at": datetime.now(timezone.utc).isoformat()}, + ) + + # Frames stay resident until the cache evicts them, so drop them now — + # otherwise deleting from disk frees no memory for up to 5 minutes. + try: + from routers.replay import evict_cached_session + evict_cached_session(year, round_num, session_type) + except Exception as e: + logger.warning(f"Could not evict replay cache for {base}: {e}") + + logger.info(f"Deleted {base} — freed {freed} bytes") + return freed diff --git a/backend/services/retention.py b/backend/services/retention.py new file mode 100644 index 0000000..c838adf --- /dev/null +++ b/backend/services/retention.py @@ -0,0 +1,280 @@ +"""Storage retention: delete stored sessions older than a user-set age. + +The setting is stored alongside the data (so it works for both the local and R2 +backends) and is enforced by a background sweep. Saving a setting runs the sweep +immediately, so applying a policy also cleans up everything already too old. + +The sweep deliberately does not live in auto_precompute_loop: that loop only +runs Fri-Mon, and isn't started at all when AUTO_PRECOMPUTE=off — which is +exactly the setup most likely to want retention. +""" + +from __future__ import annotations + +import asyncio +import calendar +import logging +import traceback +from datetime import datetime, timedelta, timezone + +from services import storage + +logger = logging.getLogger("retention") + +CONFIG_PATH = "config/retention.json" + +# How often the sweep runs once the app is up. Retention is measured in weeks +# and months, so checking more often than daily would delete the same files +# on the same day, just sooner in the day. +SWEEP_INTERVAL = 24 * 60 * 60 +STARTUP_DELAY = 60 + +UNITS = ("weeks", "months") + +# Floor on the threshold. Without it a mistyped "1 week" -> "1" could turn into +# a rolling wipe of everything the moment it is saved. +MIN_WEEKS = 1 + +DEFAULTS = {"enabled": False, "amount": 6, "unit": "months"} + + +class RetentionError(ValueError): + """Invalid retention settings.""" + + +# State of the running/last sweep, polled by the UI so a large delete shows +# progress instead of a request that just hangs. +# state is "idle" | "running" | "done" | "error" +_sweep_status: dict = {"state": "idle", "message": "", "done": 0, "total": 0, + "count": 0, "freed_bytes": 0} + + +def get_sweep_status() -> dict: + return dict(_sweep_status) + + +def _set_status(**fields) -> None: + _sweep_status.update(fields) + + +def cutoff_for(amount: int, unit: str, now: datetime | None = None) -> datetime: + """The date before which sessions are considered expired.""" + now = now or datetime.now(timezone.utc) + if unit not in UNITS: + raise RetentionError(f"unit must be one of {UNITS}") + if amount < 1: + raise RetentionError("amount must be at least 1") + if unit == "weeks": + if amount < MIN_WEEKS: + raise RetentionError(f"retention must be at least {MIN_WEEKS} week(s)") + return now - timedelta(weeks=amount) + month, year = now.month - amount, now.year + while month <= 0: + month += 12 + year -= 1 + return now.replace( + year=year, month=month, day=min(now.day, calendar.monthrange(year, month)[1]) + ) + + +def parse_utc(value: str | None) -> datetime | None: + """Parse a stored session date into an aware UTC datetime, or None.""" + if not value: + return None + try: + dt = datetime.fromisoformat(value.replace("Z", "+00:00")) + except (ValueError, TypeError): + return None + return dt.replace(tzinfo=timezone.utc) if dt.tzinfo is None else dt + + +def session_index() -> tuple[list[dict], int]: + """Every fully-stored session with its size and date. Returns (sessions, total).""" + from services.f1_data import SESSION_NAME_TO_TYPE + + try: + sizes = storage.list_sizes("sessions") + except Exception: + sizes = {} + + totals: dict[tuple[int, int, str], int] = {} + complete: set[tuple[int, int, str]] = set() + for key, size in sizes.items(): + parts = key.split("/") + # sessions/{year}/{round}/{type}/{file...} + if len(parts) < 5 or parts[0] != "sessions": + continue + try: + ident = (int(parts[1]), int(parts[2]), parts[3]) + except ValueError: + continue + totals[ident] = totals.get(ident, 0) + size + if parts[4] == "replay.json": + complete.add(ident) + + schedules: dict[int, dict] = {} + sessions: list[dict] = [] + for ident in sorted(complete): + year, rnd, code = ident + if year not in schedules: + schedules[year] = storage.get_json(f"seasons/{year}/schedule.json") or {} + event = next( + (e for e in schedules[year].get("events", []) if e.get("round_number") == rnd), + None, + ) + date_utc = None + if event: + for sess in event.get("sessions", []): + if SESSION_NAME_TO_TYPE.get(sess.get("name", "")) == code: + date_utc = sess.get("date_utc") + break + sessions.append({ + "year": year, + "round": rnd, + "type": code, + "event_name": (event or {}).get("event_name", f"Round {rnd}"), + "date_utc": date_utc, + "size_bytes": totals.get(ident, 0), + }) + + sessions.sort(key=lambda s: (s["date_utc"] or "", s["year"], s["round"]), reverse=True) + return sessions, sum(s["size_bytes"] for s in sessions) + + +def get_retention() -> dict: + stored = storage.get_json(CONFIG_PATH) + config = dict(DEFAULTS) + if isinstance(stored, dict): + config.update({k: stored[k] for k in DEFAULTS if k in stored}) + config["last_run"] = stored.get("last_run") + config["last_freed_bytes"] = stored.get("last_freed_bytes") + config["last_deleted"] = stored.get("last_deleted") + return config + + +def save_retention(enabled: bool, amount: int, unit: str) -> dict: + """Validate and persist the retention setting.""" + cutoff_for(amount, unit) # raises RetentionError on bad input + config = get_retention() + config.update({"enabled": bool(enabled), "amount": int(amount), "unit": unit}) + storage.put_json(CONFIG_PATH, config) + return config + + +def expired_sessions(amount: int, unit: str) -> list[dict]: + """Stored sessions older than the given age, oldest first.""" + cutoff = cutoff_for(amount, unit) + sessions, _ = session_index() + stale = [] + for s in sessions: + dt = parse_utc(s["date_utc"]) + if dt is not None and dt < cutoff: + stale.append(s) + stale.sort(key=lambda s: s["date_utc"] or "") + return stale + + +def sweep(amount: int, unit: str, dry_run: bool = False) -> dict: + """Delete every stored session older than the given age.""" + stale = expired_sessions(amount, unit) + if dry_run: + return { + "dry_run": True, + "count": len(stale), + "freed_bytes": sum(s["size_bytes"] for s in stale), + "sessions": stale, + } + + # Bulk delete rather than one call per session: on R2 a per-session loop is + # several sequential round trips each, which for a large first sweep means + # minutes with the request held open. + # + # No tombstones here. They exist to stop auto_precompute re-downloading a + # session the user removed, and it only ever looks at the last 7 days — + # retention never deletes anything newer than a week. + prefixes = [f"sessions/{s['year']}/{s['round']}/{s['type']}" for s in stale] + freed, deleted = 0, 0 + try: + _set_status(state="running", message="Finding files to delete…") + sizes = storage.keys_under(prefixes) + keys = list(sizes) + + def progress(done: int, total: int): + _set_status( + state="running", + message=f"Deleting {len(stale)} session{'' if len(stale) == 1 else 's'}…", + done=done, + total=total, + ) + + progress(0, len(keys)) + storage.delete_keys(keys, on_progress=progress) + freed = sum(sizes.values()) + deleted = len(stale) + except Exception as e: + logger.error(f"Retention sweep failed: {e}") + _set_status(state="error", message="Delete failed.") + + for s in stale: + try: + from routers.replay import evict_cached_session + evict_cached_session(s["year"], s["round"], s["type"]) + except Exception: + pass + + if deleted: + logger.info(f"Retention: deleted {deleted} session(s), freed {freed} bytes") + + config = get_retention() + config.update({ + "last_run": datetime.now(timezone.utc).isoformat(), + "last_freed_bytes": freed, + "last_deleted": deleted, + }) + storage.put_json(CONFIG_PATH, config) + + if _sweep_status.get("state") != "error": + _set_status( + state="done", + message=f"Deleted {deleted} session{'' if deleted == 1 else 's'}.", + count=deleted, + freed_bytes=freed, + ) + + return {"dry_run": False, "count": deleted, "freed_bytes": freed, "sessions": stale} + + +async def start_sweep(amount: int, unit: str) -> dict: + """Run a sweep in the background so the UI can poll for progress.""" + _sweep_status.clear() + _sweep_status.update({"state": "running", "message": "Starting…", + "done": 0, "total": 0, "count": 0, "freed_bytes": 0}) + + async def run(): + try: + await asyncio.to_thread(sweep, amount, unit) + except Exception as e: + logger.error(f"Sweep task failed: {e}") + _set_status(state="error", message="Delete failed.") + + asyncio.create_task(run()) + return get_sweep_status() + + +async def retention_loop(): + """Apply the retention policy shortly after startup, then once a day.""" + logger.info("Retention sweep task started") + await asyncio.sleep(STARTUP_DELAY) + + while True: + try: + config = get_retention() + if config.get("enabled"): + await asyncio.to_thread(sweep, config["amount"], config["unit"]) + else: + logger.debug("Retention disabled, skipping sweep") + except Exception as e: + logger.error(f"Retention sweep failed: {e}") + traceback.print_exc() + + await asyncio.sleep(SWEEP_INTERVAL) diff --git a/backend/services/storage.py b/backend/services/storage.py index ef20d09..d877855 100644 --- a/backend/services/storage.py +++ b/backend/services/storage.py @@ -14,6 +14,7 @@ import json import logging import os +import shutil from pathlib import Path from functools import lru_cache @@ -69,6 +70,32 @@ def _local_list_sizes(prefix: str) -> dict[str, int]: return sizes +def _local_delete(path: str) -> int: + filepath = _data_dir() / path + if not filepath.is_file(): + return 0 + size = filepath.stat().st_size + filepath.unlink() + return size + + +def _local_delete_prefix(prefix: str) -> int: + base = _data_dir() / prefix + if not base.exists(): + return 0 + if base.is_file(): + return _local_delete(prefix) + freed = sum(p.stat().st_size for p in base.rglob("*") if p.is_file()) + shutil.rmtree(base) + # Drop now-empty parents so season/round dirs don't linger in listings. + data_dir = _data_dir().resolve() + parent = base.parent + while parent.resolve() != data_dir and parent.is_dir() and not any(parent.iterdir()): + parent.rmdir() + parent = parent.parent + return freed + + # --------------------------------------------------------------------------- # R2 backend # --------------------------------------------------------------------------- @@ -167,6 +194,59 @@ def _r2_list_sizes(prefix: str) -> dict[str, int]: return sizes +def _r2_delete(path: str) -> int: + client = _get_r2_client() + key = _r2_key(path) + try: + size = client.head_object(Bucket=_r2_bucket(), Key=key)["ContentLength"] + except Exception: + return 0 + client.delete_object(Bucket=_r2_bucket(), Key=key) + return size + + +def _r2_delete_prefixes(prefixes: list[str]) -> int: + """Delete many prefixes with one listing and batched deletes. + + Deleting a hundred sessions one at a time is hundreds of sequential + round trips; this keeps it to a single listing plus one call per 1000 keys. + """ + client = _get_r2_client() + bucket = _r2_bucket() + + wanted = tuple(p if p.endswith("/") else p + "/" for p in prefixes) + # List once from the deepest shared root rather than per prefix. + root = os.path.commonprefix(list(wanted)) + root = root[: root.rfind("/") + 1] if "/" in root else "" + sizes = {k: v for k, v in _r2_list_sizes(root).items() if k.startswith(wanted)} + if not sizes: + return 0 + + keys = list(sizes) + for i in range(0, len(keys), 1000): + client.delete_objects( + Bucket=bucket, + Delete={"Objects": [{"Key": k} for k in keys[i:i + 1000]]}, + ) + return sum(sizes.values()) + + +def _r2_delete_prefix(prefix: str) -> int: + client = _get_r2_client() + bucket = _r2_bucket() + sizes = _r2_list_sizes(prefix) + if not sizes: + return 0 + keys = list(sizes) + # delete_objects caps at 1000 keys per call. + for i in range(0, len(keys), 1000): + client.delete_objects( + Bucket=bucket, + Delete={"Objects": [{"Key": k} for k in keys[i:i + 1000]]}, + ) + return sum(sizes.values()) + + # --------------------------------------------------------------------------- # Public API - delegates to the configured backend # --------------------------------------------------------------------------- @@ -205,3 +285,88 @@ def list_sizes(prefix: str) -> dict[str, int]: if _mode() == "r2": return _r2_list_sizes(prefix) return _local_list_sizes(prefix) + + +def delete(path: str) -> int: + """Delete a single stored file. Returns the number of bytes freed.""" + if not path or path.strip("/") == "": + raise ValueError("refusing to delete an empty path") + if _mode() == "r2": + return _r2_delete(path) + return _local_delete(path) + + +def delete_prefix(prefix: str) -> int: + """Delete every file under *prefix*. Returns the number of bytes freed. + + Guarded against empty prefixes and traversal so a caller can never be + tricked into wiping the whole data directory or bucket. + """ + cleaned = (prefix or "").strip("/") + if not cleaned or ".." in cleaned.split("/"): + raise ValueError(f"refusing to delete unsafe prefix: {prefix!r}") + if _mode() == "r2": + return _r2_delete_prefix(cleaned) + return _local_delete_prefix(cleaned) + + +def delete_prefixes(prefixes: list[str]) -> int: + """Delete every file under each of *prefixes*. Returns bytes freed.""" + cleaned = [] + for prefix in prefixes: + c = (prefix or "").strip("/") + if not c or ".." in c.split("/"): + raise ValueError(f"refusing to delete unsafe prefix: {prefix!r}") + cleaned.append(c) + if not cleaned: + return 0 + if _mode() == "r2": + return _r2_delete_prefixes(cleaned) + return sum(_local_delete_prefix(c) for c in cleaned) + + +def keys_under(prefixes: list[str]) -> dict[str, int]: + """Every stored key beneath any of *prefixes*, with sizes. + + Does a single listing so a caller can delete in progress-reporting batches + without paying for a fresh listing per batch. + """ + cleaned = [] + for prefix in prefixes: + c = (prefix or "").strip("/") + if not c or ".." in c.split("/"): + raise ValueError(f"refusing to list unsafe prefix: {prefix!r}") + cleaned.append(c if c.endswith("/") else c + "/") + if not cleaned: + return {} + + root = os.path.commonprefix(cleaned) + root = root[: root.rfind("/") + 1] if "/" in root else "" + wanted = tuple(cleaned) + return {k: v for k, v in list_sizes(root).items() if k.startswith(wanted)} + + +def delete_keys(keys: list[str], on_progress=None) -> None: + """Delete exactly these keys. on_progress(done, total) fires per batch.""" + total = len(keys) + if not total: + return + r2 = _mode() == "r2" + client = _get_r2_client() if r2 else None + bucket = _r2_bucket() if r2 else None + + step = 1000 if r2 else 200 + for i in range(0, total, step): + batch = keys[i:i + step] + if r2: + client.delete_objects( + Bucket=bucket, Delete={"Objects": [{"Key": k} for k in batch]} + ) + else: + for k in batch: + try: + (_data_dir() / k).unlink() + except FileNotFoundError: + pass + if on_progress: + on_progress(min(i + step, total), total) diff --git a/docker-compose.yml b/docker-compose.yml index 11e3b92..23b31e9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,10 +2,13 @@ services: f1timing: build: . # Uses root Dockerfile (unified FE+BE) ports: - - "${PORT:-8000}:${PORT:-8000}" + - "${PORT:-8000}:8000" env_file: .env environment: - PORT: ${PORT:-8000} + # The container always listens on 8000 so reverse proxies have a stable + # target. PORT sets the published host port only (overrides any PORT + # in .env, which would otherwise move the port proxies point at). + PORT: 8000 volumes: - f1data:/data - f1cache:/data/fastf1-cache diff --git a/frontend/src/app/replay/page.tsx b/frontend/src/app/replay/page.tsx index 34e46c0..516be4c 100644 --- a/frontend/src/app/replay/page.tsx +++ b/frontend/src/app/replay/page.tsx @@ -389,6 +389,14 @@ export default function ReplayPage() { return (
+ {/* Connection lost — the socket retries on its own in the background */} + {replay.reconnecting && ( +
+ + Connection lost — reconnecting… +
+ )} + {/* Banner */} {!fullscreen && sessionData && ( (null); const [selectedKey, setSelectedKey] = useState(null); const [menuOpen, setMenuOpen] = useState(false); + const [storageOpen, setStorageOpen] = useState(false); const [navigating, setNavigating] = useState(false); useEffect(() => { setNavigating(false); @@ -166,6 +170,7 @@ export default function SessionPicker() { const [ctxMenu, setCtxMenu] = useState(null); const [reprocessing, setReprocessing] = useState>(new Set()); const [reprocessModal, setReprocessModal] = useState<{ label: string; key: string; state: "running" | "done" | "error"; message: string } | null>(null); + const [deleteModal, setDeleteModal] = useState<{ label: string; year: number; round: number; code: string; sizeLabel: string | null; state: "confirm" | "deleting" | "done" | "error"; message: string } | null>(null); const longPressRef = useRef | null>(null); const suppressClickRef = useRef(false); const ctxMenuRef = useRef(null); @@ -227,6 +232,27 @@ export default function SessionPicker() { } } + async function confirmDelete() { + if (!deleteModal) return; + const m = deleteModal; + setDeleteModal({ ...m, state: "deleting" }); + try { + const res = await authFetch( + `/api/sessions/${m.year}/${m.round}?type=${m.code}`, + { method: "DELETE" }, + ); + if (!res.ok) throw new Error(); + const body = await res.json(); + setDeleteModal({ + ...m, + state: "done", + message: `Freed ${formatSize(body.freed_bytes) || "0 KB"}.`, + }); + } catch { + setDeleteModal({ ...m, state: "error", message: "Failed to delete this session's data." }); + } + } + // Close the context menu on outside click, scroll, resize, or Escape useEffect(() => { if (!ctxMenu) return; @@ -373,7 +399,7 @@ export default function SessionPicker() { const sKey = `${year}_${evt.round_number}_${code}`; const busy = reprocessing.has(sKey); const openMenu = (x: number, y: number) => - setCtxMenu({ x, y, href, label: session.name, key: sKey, year, round: evt.round_number, code }); + setCtxMenu({ x, y, href, label: session.name, key: sKey, year, round: evt.round_number, code, precomputed: !!session.precomputed, sizeBytes: session.size_bytes }); return (
{localTime && ( @@ -476,6 +502,85 @@ export default function SessionPicker() { > ↻ Reprocess + {ctxMenu.precomputed && ( + + )} +
+ )} + + {storageOpen && setStorageOpen(false)} />} + + {/* Delete confirmation */} + {deleteModal && ( +
+
+

+ {deleteModal.state === "done" ? "Data deleted" : `Delete ${deleteModal.label} data?`} +

+ {deleteModal.state === "confirm" && ( + <> +

+ Removes the stored data for this session + {deleteModal.sizeLabel ? `, freeing ${deleteModal.sizeLabel}` : ""}. Open the + session to download it again. +

+
+ + +
+ + )} + {deleteModal.state === "deleting" && ( +
+ + Deleting… +
+ )} + {(deleteModal.state === "done" || deleteModal.state === "error") && ( + <> +

+ {deleteModal.message} +

+
+ +
+ + )} +
)} @@ -536,6 +641,12 @@ export default function SessionPicker() { > About + {/* Mobile: hamburger menu */}
)}
diff --git a/frontend/src/components/StorageManager.tsx b/frontend/src/components/StorageManager.tsx new file mode 100644 index 0000000..1f426b6 --- /dev/null +++ b/frontend/src/components/StorageManager.tsx @@ -0,0 +1,388 @@ +"use client"; + +import { useState, useEffect, useCallback } from "react"; +import { getToken } from "@/lib/auth"; +import { apiUrl } from "@/lib/api"; + +interface StoredSession { + year: number; + round: number; + type: string; + event_name: string; + date_utc: string | null; + size_bytes: number; +} + +interface Retention { + enabled: boolean; + amount: number; + unit: Unit; + last_run?: string | null; + last_freed_bytes?: number | null; + last_deleted?: number | null; +} + +interface StorageResponse { + total_bytes: number; + session_count: number; + sessions: StoredSession[]; + retention: Retention; +} + +interface SweepStatus { + state: "idle" | "running" | "done" | "error"; + message: string; + done: number; + total: number; + count: number; + freed_bytes: number; +} + +interface SweepResult { + count: number; + freed_bytes: number; + sessions?: StoredSession[]; +} + +type Unit = "weeks" | "months"; + +function formatSize(bytes?: number | null): string { + if (!bytes || bytes <= 0) return "0 KB"; + const gb = bytes / (1024 * 1024 * 1024); + if (gb >= 1) return `${gb.toFixed(2)} GB`; + const mb = bytes / (1024 * 1024); + if (mb >= 1) return `${mb.toFixed(1)} MB`; + return `${Math.max(1, Math.round(bytes / 1024))} KB`; +} + +function formatWhen(iso?: string | null): string | null { + if (!iso) return null; + const then = new Date(iso).getTime(); + if (Number.isNaN(then)) return null; + const mins = Math.round((Date.now() - then) / 60000); + if (mins < 1) return "just now"; + if (mins < 60) return `${mins} min ago`; + const hours = Math.round(mins / 60); + if (hours < 24) return `${hours} hour${hours === 1 ? "" : "s"} ago`; + const days = Math.round(hours / 24); + return `${days} day${days === 1 ? "" : "s"} ago`; +} + +function formatDate(iso?: string | null): string { + if (!iso) return ""; + const d = new Date(iso.replace(" ", "T")); + if (Number.isNaN(d.getTime())) return ""; + return d.toLocaleDateString(undefined, { + day: "numeric", + month: "short", + year: "numeric", + }); +} + +function authFetch(path: string, init?: RequestInit) { + const token = getToken(); + return fetch(apiUrl(path), { + ...init, + headers: { + ...(init?.headers || {}), + ...(token ? { Authorization: `Bearer ${token}` } : {}), + }, + }); +} + +export default function StorageManager({ onClose }: { onClose: () => void }) { + const [data, setData] = useState(null); + const [loading, setLoading] = useState(true); + + const [enabled, setEnabled] = useState(false); + // Held as a string so the field can be cleared and overtyped; parsed on use. + const [amount, setAmount] = useState("6"); + const [unit, setUnit] = useState("months"); + const amountNum = Math.max(1, parseInt(amount, 10) || 1); + const unitLabel = amountNum === 1 ? unit.slice(0, -1) : unit; + + const [confirm, setConfirm] = useState(null); + const [progress, setProgress] = useState(null); + const [busy, setBusy] = useState(false); + const [result, setResult] = useState(null); + const [error, setError] = useState(null); + + const load = useCallback(async () => { + setLoading(true); + try { + const res = await authFetch("/api/storage"); + if (!res.ok) throw new Error(); + const body: StorageResponse = await res.json(); + setData(body); + setEnabled(body.retention.enabled); + setAmount(String(body.retention.amount)); + setUnit(body.retention.unit); + } catch { + setError("Could not read storage usage."); + } finally { + setLoading(false); + } + }, []); + + useEffect(() => { + load(); + }, [load]); + + useEffect(() => { + const onKey = (e: KeyboardEvent) => { + if (e.key === "Escape") onClose(); + }; + document.addEventListener("keydown", onKey); + return () => document.removeEventListener("keydown", onKey); + }, [onClose]); + + const saved = data?.retention; + const dirty = + !!saved && + (enabled !== saved.enabled || amountNum !== saved.amount || unit !== saved.unit); + + // A pending confirmation is only valid for the values it was taken against. + useEffect(() => { + setConfirm(null); + setResult(null); + }, [enabled, amount, unit]); + + const query = `enabled=${enabled}&amount=${amountNum}&unit=${unit}`; + + async function requestSave() { + setBusy(true); + setError(null); + try { + // Turning it off deletes nothing, so skip straight to saving. + if (!enabled) { + const res = await authFetch(`/api/storage/retention?${query}`, { method: "PUT" }); + if (!res.ok) throw new Error(); + setResult("Automatic deletion turned off."); + await load(); + return; + } + const res = await authFetch(`/api/storage/retention?${query}&dry_run=true`, { + method: "PUT", + }); + if (!res.ok) throw new Error(); + const body: SweepResult = await res.json(); + if (body.count === 0) { + // Nothing to remove today — just save the policy. + const apply = await authFetch(`/api/storage/retention?${query}`, { method: "PUT" }); + if (!apply.ok) throw new Error(); + setResult("Saved. Nothing stored is that old yet."); + await load(); + return; + } + setConfirm(body); + } catch { + setError("Could not save the retention setting."); + } finally { + setBusy(false); + } + } + + async function applySave() { + setBusy(true); + setError(null); + setConfirm(null); + setProgress({ state: "running", message: "Starting…", done: 0, total: 0, count: 0, freed_bytes: 0 }); + try { + const res = await authFetch(`/api/storage/retention?${query}`, { method: "PUT" }); + if (!res.ok) throw new Error(); + + // The sweep runs server-side; poll so a large delete shows progress. + for (;;) { + await new Promise((r) => setTimeout(r, 400)); + const s: SweepStatus = await authFetch("/api/storage/retention/status").then((r) => r.json()); + setProgress(s); + if (s.state === "done") { + setResult(`Deleted ${s.count} session${s.count === 1 ? "" : "s"}, freeing ${formatSize(s.freed_bytes)}.`); + break; + } + if (s.state === "error") { + setError(s.message || "Delete failed."); + break; + } + } + setProgress(null); + await load(); + } catch { + setProgress(null); + setError("Failed to apply the retention setting."); + } finally { + setBusy(false); + } + } + + const lastRun = formatWhen(saved?.last_run); + + return ( +
+
+
+

Storage

+ {loading ? ( +

Reading usage…

+ ) : ( +

+ {data?.session_count || 0} session{data?.session_count === 1 ? "" : "s"} stored + {" · "} + {formatSize(data?.total_bytes)} +

+ )} +
+ + {/* Retention policy — hidden until usage is known, so the threshold + can't be set against a total that hasn't loaded yet. */} + {loading ? ( +
+ + Calculating current storage… +
+ ) : ( +
+ + +
+
+ Delete sessions older than + setAmount(e.target.value.replace(/[^0-9]/g, "").slice(0, 3))} + onFocus={(e) => e.target.select()} + onBlur={() => { if (!amount) setAmount("1"); }} + aria-label="Age threshold" + className="w-14 px-2 py-1.5 bg-f1-dark border border-f1-border rounded text-white text-sm text-center" + /> + +
+

+ Checked daily. Open a session to download it again. +

+
+ + {saved?.enabled && lastRun && !dirty && ( +

+ Last run {lastRun} + {saved.last_deleted + ? ` — removed ${saved.last_deleted} session${saved.last_deleted === 1 ? "" : "s"}, freeing ${formatSize(saved.last_freed_bytes)}.` + : " — nothing was old enough to remove."} +

+ )} + + {dirty && !confirm && ( + + )} + + {confirm && ( +
+

+ Deletes {confirm.count} session + {confirm.count === 1 ? "" : "s"} now, freeing{" "} + {formatSize(confirm.freed_bytes)}. Runs + daily for sessions older than{" "} + {amountNum} {unitLabel}. +

+ + {!!confirm.sessions?.length && ( +
+ {confirm.sessions.map((s) => ( +
+ + {s.event_name} + {s.type} + + + {formatDate(s.date_utc)} · {formatSize(s.size_bytes)} + +
+ ))} +
+ )} +
+ + +
+
+ )} + + {progress && ( +
+
+ + {progress.message} +
+ {progress.total > 0 && ( + <> +
+
+
+

+ {progress.done} of {progress.total} files +

+ + )} +
+ )} + + {result &&

{result}

} + {error &&

{error}

} +
+ )} + +
+ +
+
+
+ ); +} diff --git a/frontend/src/hooks/useReplaySocket.ts b/frontend/src/hooks/useReplaySocket.ts index effc7b6..5f6beda 100644 --- a/frontend/src/hooks/useReplaySocket.ts +++ b/frontend/src/hooks/useReplaySocket.ts @@ -84,6 +84,7 @@ export interface QualiPhaseInfo { interface ReplayState { connected: boolean; + reconnecting: boolean; ready: boolean; loading: boolean; playing: boolean; @@ -99,8 +100,16 @@ interface ReplayState { export function useReplaySocket(year: number, round: number, sessionType: string = "R") { const wsRef = useRef(null); + // Reconnect bookkeeping. Without this a dropped socket — a proxy idle + // timeout, a network blip — freezes the UI permanently with no feedback. + const unmountedRef = useRef(false); + const attemptRef = useRef(0); + const retryTimerRef = useRef | null>(null); + const lastTimestampRef = useRef(0); + const wasPlayingRef = useRef(false); const [state, setState] = useState({ connected: false, + reconnecting: false, ready: false, loading: true, playing: false, @@ -115,18 +124,28 @@ export function useReplaySocket(year: number, round: number, sessionType: string }); useEffect(() => { + unmountedRef.current = false; + attemptRef.current = 0; + lastTimestampRef.current = 0; + wasPlayingRef.current = false; + + const connect = () => { const url = wsUrl(`/ws/replay/${year}/${round}?type=${sessionType}`); const ws = new WebSocket(url); wsRef.current = ws; ws.onopen = () => { - setState((s) => ({ ...s, connected: true })); + attemptRef.current = 0; + setState((s) => ({ ...s, connected: true, reconnecting: false })); }; ws.onmessage = (event) => { const msg = JSON.parse(event.data); switch (msg.type) { + case "ping": + // Server keep-alive so proxies don't time the socket out. Nothing to do. + break; case "status": setState((s) => ({ ...s, loading: true, statusMessage: msg.message || null })); break; @@ -140,12 +159,15 @@ export function useReplaySocket(year: number, round: number, sessionType: string totalLaps: msg.total_laps, qualiPhases: msg.quali_phases || [], })); - // Request first frame so cars are visible before play + // Seek to where we were so a reconnect resumes in place; 0 on a + // first connect, which also makes cars visible before play. if (ws.readyState === WebSocket.OPEN) { - ws.send("seek:0"); + ws.send(`seek:${lastTimestampRef.current}`); + if (wasPlayingRef.current) ws.send("play"); } break; case "frame": + lastTimestampRef.current = msg.timestamp; setState((s) => ({ ...s, frame: { @@ -164,6 +186,7 @@ export function useReplaySocket(year: number, round: number, sessionType: string })); break; case "finished": + wasPlayingRef.current = false; setState((s) => ({ ...s, playing: false, finished: true })); break; case "error": @@ -173,15 +196,28 @@ export function useReplaySocket(year: number, round: number, sessionType: string }; ws.onerror = () => { - setState((s) => ({ ...s, error: "WebSocket connection error", loading: false })); + // onclose always follows, and handles retrying. + setState((s) => ({ ...s, loading: false })); }; ws.onclose = () => { - setState((s) => ({ ...s, connected: false })); + wsRef.current = null; + if (unmountedRef.current) return; + + // Retry with backoff, capped, so a transient drop recovers on its own. + const delay = Math.min(1000 * 2 ** attemptRef.current, 15000); + attemptRef.current += 1; + setState((s) => ({ ...s, connected: false, reconnecting: true })); + retryTimerRef.current = setTimeout(connect, delay); }; + }; + + connect(); return () => { - ws.close(); + unmountedRef.current = true; + if (retryTimerRef.current) clearTimeout(retryTimerRef.current); + wsRef.current?.close(); }; }, [year, round, sessionType]); @@ -192,11 +228,13 @@ export function useReplaySocket(year: number, round: number, sessionType: string }, []); const play = useCallback(() => { + wasPlayingRef.current = true; send("play"); setState((s) => ({ ...s, playing: true, finished: false })); }, [send]); const pause = useCallback(() => { + wasPlayingRef.current = false; send("pause"); setState((s) => ({ ...s, playing: false })); }, [send]); @@ -217,6 +255,8 @@ export function useReplaySocket(year: number, round: number, sessionType: string }, [send]); const reset = useCallback(() => { + wasPlayingRef.current = false; + lastTimestampRef.current = 0; send("reset"); setState((s) => ({ ...s, playing: false, finished: false })); }, [send]);