pyproject.toml

[build-system]
requires = ["setuptools>=68", "wheel"]
build-backend = "setuptools.build_meta"

[project]
name = "evalrx"
dynamic = ["version"]
description = "Failure case analysis in the era of LLM."
readme = "README.md"
license = { file = "LICENSE" }
requires-python = ">=3.10"
keywords = ["llm", "vlm", "interpretability", "failure-analysis", "explainability"]
classifiers = [
    "Development Status :: 3 - Alpha",
    "Intended Audience :: Science/Research",
    "License :: CC0 1.0 Universal (CC0 1.0) Public Domain Dedication",
    "Programming Language :: Python :: 3",
    "Programming Language :: Python :: 3.10",
    "Programming Language :: Python :: 3.11",
    "Programming Language :: Python :: 3.12",
    "Topic :: Scientific/Engineering :: Artificial Intelligence",
]
# Light core: pure-API failure analysis installs WITHOUT torch.
# White-box / local inference lives in the [local] extra; see below.
dependencies = [
    "pyyaml>=6.0",
    "numpy>=1.24",
]

[project.optional-dependencies]
# Local white-box models + internals capture (the heavy path).
#
# The <5 cap excluded the benchmark's own headline model: Qwen3.5 declares
# model_type "qwen3_5", which no 4.x release knows, so installing this extra on
# a box that already had transformers 5 DOWNGRADED it and hf_local then refused
# the checkpoint with "Transformers does not recognize this architecture".
# Verified on 5.15: the spec loads in 10s and reports the full white-box set
# (attention / hidden_states / logits / logprobs).
local = [
    "torch>=2.0.0",
    "transformers>=4.51.0",
    "accelerate>=0.26.0",
]
# L4 fix tier (parameter-space repair): LoRA fine-tuning executor. Separate from
# `local` because it's only needed when FixAgent is run with a finetune_pool —
# most `local` (inference-only) users never touch this path.
finetune = [
    "peft>=0.10",
]
# JAX white-box backend (jax_local, docs/design_jax_backend.md). Google DeepMind's
# gemma library drives the Gemma 4 reference adapter (Flax Linen; Python >= 3.12
# upstream). Accelerator wheels are a separate choice: `pip install "jax[cuda12]"`
# (GPU) or `pip install "jax[tpu]"` (Cloud TPU; Colab TPU runtimes ship it, see
# tools/colab/).
# Captured arrays cross into the analyzers as torch tensors, so a CPU torch is
# enough here (`--extra-index-url https://download.pytorch.org/whl/cpu`).
jax = [
    "jax>=0.7",
    "flax>=0.10",
    "gemma>=4.0.1; python_version >= '3.12'",
    "torch>=2.0.0",
]
# API / black-box models. The OpenAI-compatible client is optional; you can also
# inject an existing call_vision_api via RuntimeConfig(generate_fn=...).
api = [
    "openai>=1.0",
]
# Interpretability toolchains for the white-box analyzers (Stage 2+).
interp = [
    "captum>=0.7",
    "inseq>=0.6",
    "nnsight>=0.3",
]
# Benchmark dataset loaders that pull from the HuggingFace Hub (e.g. Spatial457).
data = [
    "datasets>=2.0",
    "pillow>=9.0",
    # The text-LLM benchmark tasks (evalrx.benchmark.tasks.llm) load
    # examples/dataset_selection/llm_band_probe/band_locate.py, which imports
    # requests at module level to page through the HF datasets-server.
    "requests>=2.28",
]
# Machine-readable M1-M5 stage contract (evalrx.contract): runtime validation
# of what crosses each stage boundary + JSON Schema / TypeScript generation.
contract = ["pydantic>=2.7"]
viz = ["matplotlib>=3.7"]
# Langfuse is optional so local/offline evaluations keep the light core.  When
# installed, every RunLogger event is persisted through the local outbox and
# mirrored to Langfuse on flush.
observability = ["langfuse>=4.7"]
# Completed-run dynamic report server. The React renderer is pre-built and
# shipped as package data; Node is only needed by contributors changing it.
ui = [
    "fastapi>=0.115",
    "uvicorn>=0.30",
    # fastapi needs this for Form/File(...) params (POST /api/upload) but
    # doesn't declare it as a hard dependency -- without it those routes
    # 500 at request time (import succeeds, so this is easy to miss).
    "python-multipart>=0.0.9",
]
# Inferential stats for the outcome-driver-analysis agent skill's Python path
# (logistic/GLM fits with real inference, VIF, calibration, ROC).
stats = [
    "statsmodels>=0.14",
    "scikit-learn>=1.3",
]
# Failure-mode clustering (evalrx.analysis.failure_modes). scikit-learn
# alone gets TF-IDF + Agglomerative clustering; hdbscan adds density-based
# clustering (no fixed cluster count). Neither is required — a pure-numpy
# fallback (hashing vectorizer + cosine-greedy grouping) always works.
cluster = [
    "scikit-learn>=1.3",
    "hdbscan>=0.8.33",
    "langfuse>=4.7",
]
# The former Streamlit dashboard and upload workbench are no longer supported.
gemini = ["google-genai>=1.0"]
all = [
    "torch>=2.0.0",
    "transformers>=4.51.0",
    "accelerate>=0.26.0",
    "peft>=0.10",
    "openai>=1.0",
    "captum>=0.7",
    "inseq>=0.6",
    "nnsight>=0.3",
    "matplotlib>=3.7",
    "statsmodels>=0.14",
    "scikit-learn>=1.3",
    "hdbscan>=0.8.33",
]
dev = [
    "pytest>=8.0",
    "pytest-cov>=4.0",
    "ruff>=0.4",
    "mypy>=1.8",
    # Validate run_log.jsonl against the published schema (log_schema.py).
    "jsonschema>=4.0",
]

[project.urls]
Homepage = "https://github.com/evalvitals/evalrx"
Documentation = "https://evalvitals.github.io/evalrx/"
Repository = "https://github.com/evalvitals/evalrx"
Issues = "https://github.com/evalvitals/evalrx/issues"
Changelog = "https://github.com/evalvitals/evalrx/blob/main/CHANGELOG.md"

[project.scripts]
evalrx = "evalrx.cli:main"
evalrx-explore = "evalrx.analysis.cli:main"

[tool.setuptools.dynamic]
version = {attr = "evalrx.__version__"}

[tool.setuptools.packages.find]
where = ["."]
include = ["evalrx*"]

[tool.setuptools.package-data]
# Ship the published run_log.jsonl JSON Schema so non-Python consumers can
# validate the log without installing evalrx.
"evalrx.eval_agent" = ["run_log.schema.json"]
# Ship bundled Agent Skills (e.g. nature-figure) so `pip install` carries them
# too, not just `git clone`. explore auto-applies them on claude/agy backends.
"evalrx.agent_assets" = ["skills/**/*"]
"evalrx.reporting" = ["web_dist/**/*"]

[tool.pytest.ini_options]
testpaths = ["tests"]
addopts = "-v --tb=short"
markers = [
    "gpu: requires CUDA GPU and cached model weights — skip with -m 'not gpu'",
    "slow: takes more than a few seconds on CPU",
    "network: downloads a small HF processor/tokenizer config (no GPU, no weights) — skip with -m 'not network'",
]

[tool.ruff]
line-length = 100
target-version = "py310"
exclude = ["examples/"]

[tool.ruff.lint]
select = ["E", "F", "I"]
ignore = ["E501"]

[tool.mypy]
python_version = "3.10"
ignore_missing_imports = true