pyproject.toml

[project]
name = "replicator"
version = "0.1.0"
description = "retrieval, fingerprinting, and temporary storage layer for the Cannabis Observer cluster"
readme = "README.md"
license = "MIT"
license-files = ["LICENSE"]
requires-python = ">=3.12"
dependencies = [
    # Floor is 0.40.0, and it carries no new feature: a one-minor catch-up
    # (#172), ahead of #63's `ContentFetchCommand.render` (cannobserv#602, a
    # 0.40.1 PATCH not yet cut), so that this diff does not land with a
    # behaviour change. #63 raises the patch floor itself when it starts reading
    # the field. 0.40.0's two breaks are producer-side: `ContentReplicateCommandEmit`
    # enforces the `<provider>-<role>` alias rule (cannobserv#495) and
    # `SourceRevisionObservedEmit` requires `blob_fingerprint` (cannobserv#494).
    # Replicator builds neither and decodes the canonical classes, which are
    # unchanged; `ContentReplicateCommand.credentials_alias` stays a plain `str`,
    # so an unknown alias still decodes and closes as `alias_unknown`. Both
    # `co-core-aio` and `co-core-sync` 0.40.0 require `co-core>=0.40,<0.41`, so
    # the three move together.
    #
    # **`scripts/co_core_closure_diff.py 0.39.1 0.40.0`** (#145), verbatim:
    #
    # ## Import closure, 0.39.1 → 0.40.0
    #
    # Roots `src`, `scripts`; packages `co_core`, `co_core_aio`, `co_core_sync`.
    #
    # 58 files at 0.39.1, 58 at 0.40.0. 8 not identical:
    #
    # - `version-only` co_core/__init__.py
    # - `changed` co_core/pure/models/changes.py
    # - `docstrings-only` co_core/pure/util/aliases.py
    # - `version-only` co_core_aio/__init__.py
    # - `docstrings-only` co_core_aio/fetch.py
    # - `changed` co_core_aio/transport.py
    # - `version-only` co_core_sync/__init__.py
    # - `changed` co_core_sync/drivers/transport.py
    #
    # **Requires-Dist:**
    #
    # - co_core: unchanged
    # - co_core_aio: removed `co-core<0.40,>=0.39`; added `co-core<0.41,>=0.40`
    # - co_core_sync: removed `co-core<0.40,>=0.39`; added `co-core<0.41,>=0.40`
    #
    # **Third-party imports** (at 0.40.0): `google`, `google_auth_oauthlib`, `googleapiclient`, `httpx`, `pydantic`, `redis`, `requests`, `urllib3`.
    #
    # **Dynamic imports, not followed:**
    #
    # - co_core_sync/drivers/__init__.py:89 `import_module(f"{__name__}.{module}")`
    # - co_core_sync/drivers/blobstore/__init__.py:32 `import_module(f"{__name__}.{module}")`
    #
    # Read by hand, the `changed` files:
    #
    # - `pure/models/changes.py`: the two Emit twins above, and the import of
    #   `alias_provider` / `validate_credentials_alias` that the alias check
    #   needs. Every other class is AST-identical, so no field, default or
    #   config moved under `extra="ignore"`;
    # - `co_core_aio/transport.py` (0.39.4, cannobserv#597): `RetryAsyncTransport`
    #   re-raises a raw `ssl.SSLError` as `httpx.ReadError`. In the closure only
    #   because `fetch.py` imports `build_resilient_client`. `build_fetch_client`
    #   hands `AsyncFetchDriver` its own client, whose transport is
    #   `GuardedTransport(BodyCeilingTransport(httpx.AsyncHTTPTransport()))`,
    #   so the change never runs here. **A raw `ssl.SSLError` from a TLS read
    #   still misses `handler.py`'s `except httpx.HTTPError`** and closes as
    #   `handler_error` after the delivery ceiling: unchanged by this move;
    # - `co_core_sync/drivers/transport.py` (0.39.2, cannobserv#509):
    #   `DeadlineRetry.increment` stops re-sending a POST or PATCH after a TLS
    #   error. Loaded because `co_core_sync/drivers/__init__` imports it eagerly,
    #   but it acts only on a session mounted with `mount_resilient_adapter` or
    #   `resilient_adapter`. Nothing here mounts one; `GcsBlobStore` goes through
    #   `google.cloud.storage`'s own session.
    #
    # Requires-Dist is unchanged bar the self-pins. 0.39.3's Trello transport and
    # 0.39.5's `extract_email_parts` fall outside the closure. All of
    # `co_core_aio/bus` and `pure/adapters/bus` is identical. The two dynamic
    # imports are the same two as at 0.39.1, each target named again under
    # `TYPE_CHECKING`.
    #
    # Previously:
    #
    # Floor was 0.39.0, and it carried no new feature: **two minors (0.37 -> 0.39)**
    # taken as a catch-up (#153), ahead of the capture consumer (#138), so that
    # that diff did not land with a behaviour change. #138's types
    # (`ContentCaptureCommand`, `CaptureStoredEmit` / `CaptureFailedEmit`, the
    # `CONTENT_CAPTURE` constant) arrived in 0.37.1 (cannobserv#571), so every
    # 0.39 release has them. The floor was 0.39.0, not 0.39.1: 0.39.1's diff over
    # 0.39.0 is `__version__` in the three `__init__`s and nothing else.
    #
    # **Diffing the closure, 0.37.0 → 0.39.1** (57 files, then 58): 14 not
    # identical. `pure/models/changes.py` added the capture models and their Emit
    # twins; every class that existed at 0.37.0 was AST-identical bar
    # docstrings. `from_wire`'s global table gained `content_capture`,
    # `capture_stored` and `capture_failed`, so one of those on a stream read
    # here decodes since 0.39 and falls to the `isinstance` guards: a command stream
    # still dead-letters it, under "payload is not a ... command" instead of a
    # poison frame, and `content.fetch-policy` skips it with a wrong-type warning
    # instead of a poison skip. `streams.py` added `CONTENT_CAPTURE`.
    # `pure/extract/*` (0.38.0's JSON extractor, `ignore_selectors`, the added
    # `_canonical_json.py`) loads only because `changes.py` imports
    # `extract.canonical`. The new Requires-Dist was gated on `extra == 'extract'`,
    # which isn't installed. All of `co_core_aio` was identical bar `__version__`.
    #
    # Previously:
    #
    # Floor was 0.37.0, for `FetchPolicyState.generation` and
    # `fetch_policy_supersedes` (cannobserv#566, #568; #136). `FetchPolicyMap`
    # applies every policy frame through the helper, so on 0.36.0 the worker fails
    # at import. The field alone would fail silently instead: under
    # `extra="ignore"` a 0.36 wheel drops `generation`, and every Archiver frame
    # applies in stream order — the outbox reorder the field exists to stop. 0.37
    # is a minor, but every entry in it is additive; the owner chose the minor to
    # mark the headless-Watcher contract set. `co-core-aio` and `co-core-sync`
    # 0.37.0 require `co-core>=0.37,<0.38`, so the three move together.
    #
    # **No `[extract]` extra since #142.** It pulled in five parsers —
    # beautifulsoup4, html5lib, lxml, openpyxl and pypdf, with et-xmlfile,
    # soupsieve and webencodings under them — and nothing here imports one. The
    # closure reaches `pure/extract/__init__` only through `changes.py`'s import of
    # `extract.canonical`, and none of the modules that loads imports a parser. For
    # a fetcher of arbitrary public URLs an unused HTML/PDF/XLSX parser is CVE
    # surface and nothing else.
    #
    # **Diffing the wheels covered the whole import closure**: 57 files at both
    # 0.36.0 and 0.37.0, compared by AST with docstrings removed. Six changed:
    #
    # - the three `__init__`s, `__version__` only;
    # - `pure/models/changes.py`: the field and the helper, plus Watcher models,
    #   three `WatchStatusState` fields and a `ProcessingFailureReason` value,
    #   none imported here;
    # - `pure/adapters/bus/envelope.py`: the global `from_wire` table gains five
    #   Watcher types (cannobserv#563, #564). They decode cleanly on any stream and
    #   fall to the `isinstance` guards every consume path already has;
    # - `pure/adapters/bus/streams.py`: two stream constants.
    #
    # Requires-Dist is unchanged bar the self-pins.
    #
    # Previously:
    #
    # Floor was 0.36.0, and it carried no new feature: **seventeen minors (0.19 ->
    # 0.36)** taken as a catch-up (#140), ahead of the release #136 needed, so
    # that this diff did not land with a behaviour change. 0.36.0 was the minor's
    # only release. Both `co-core-aio` and `co-core-sync` 0.36.0 require
    # `co-core>=0.36,<0.37`, so the three can't drift apart.
    #
    # **Diffing the wheels covered the whole import closure**, not just the
    # modules named here: 56 files at 0.19.6 and 57 at 0.36.0, compared by AST
    # with docstrings removed. The models and the util and bus modules are
    # identical bar docstrings: `pure/models/changes.py`, `pure/util/{blobstore,gcs}.py`
    # and `pure/adapters/bus/streams.py`. So no payload field moved under
    # `extra="ignore"`, and `FetchPolicyState`'s fields, defaults and config are
    # unchanged. The rest of `pure/adapters/bus/*`, `effects/{bus,fetch,gcs}.py`,
    # `pure/util/hashing.py` and all of `co_core_aio` are byte-identical, bar
    # `__version__`. Two changes are additive, and neither is called here:
    #
    # - `co_core_sync.drivers.blobstore.gcs` adds the pointer reconciler's verbs:
    #   `PointerEntry`, `list_pointer_entries` and `delete_pointer`
    #   (cannobserv#496).
    # - `pure/extract/*` adds XML extraction and `UnsupportedExtractionError`
    #   (cannobserv#523). It loads only because `changes.py` imports
    #   `extract.canonical`.
    #
    # Requires-Dist and the extras are unchanged bar the self-pins. Every Breaking
    # entry in the CHANGELOG across the range falls in one of three places: the
    # domain models (information, information set, jurisdiction, address,
    # event), their WordPress and co/v1 adapters, or `extraction_config_from_spec`.
    # Replicator imports or calls none of them. That includes cannobserv#549's
    # aware-only datetimes, which cover those domain models and the wp/v2
    # formatters, not the change-bus payloads.
    #
    # Previously:
    #
    # Floor was 0.19.6, for the `content.persist` contract (cannobserv#493, #114
    # step 6): `ContentPersistCommand`, `BlobPersistedEvent` / `PersistFailedEvent`
    # and their Emit twins, and the `CONTENT_PERSIST` stream constant. On 0.19.5 the
    # worker fails at import. The same release makes both blob stores hash what they
    # write (cannobserv#492, `FingerprintMismatch`), which the persist handler
    # reports; 0.19.4 and 0.19.5 are additive for everything used here.
    #
    # Previously:
    #
    # Floor was 0.19.3, for the shared content-addressed store (cannobserv#475,
    # #114): `co_core.pure.util.blobstore` (the key scheme and the `BlobStore`
    # Protocol), `co_core_sync.drivers.blobstore` (both backends, the `gcs` one
    # behind `co-core-sync[gcs]`), and `checksum="crc32c"` on every conditional
    # create — including `AsyncGcsDriver.create_if_absent`, which the replicate
    # path was already using, so the floor moves on all three packages at once.
    # Diffing the wheels: 0.19.3 adds `pure/util/blobstore.py`, the checksum
    # constant in `pure/util/gcs.py` and `sha256_chunks` in `pure/util/hashing.py`;
    # the aio driver gains the `checksum=` kwarg; `pure/models/changes.py`, the
    # effects and the bus adapters are byte-identical. On 0.19.2 the worker fails
    # at import with a ModuleNotFoundError, never reaching a running consumer.
    #
    # Previously:
    #
    # Floor was 0.19.1, for `AsyncBusConsumer.dead_letter`'s `reason=` keyword and
    # the provenance it writes onto every DLQ entry (`dlq.source_id`, `dlq.group`,
    # `dlq.consumer`, `dlq.reason` — cannobserv#474, #116), which retired the
    # `dlq_reason` / `dlq_original_id` keys `_dead_letter` added by hand. It also
    # accepts an empty field map, which lets `dead_letter_anomaly` park a trimmed
    # entry without inventing fields for it. On 0.19.0 the first dead-letter raises
    # TypeError on the keyword — at runtime, on the error path, which is why the
    # floor carries it; `tests/worker/` fails at collection instead, its conftest
    # importing `co_core.pure.adapters.bus.dead_letter`, which 0.19.0 does not
    # ship. Diffing the wheels: `dead_letter` is the only behavioural change to
    # anything imported here — `PoisonFrame`'s docstring moved too — and 0.19.1
    # otherwise touches co/v1 engagement options and `AddressModel`. 0.19.2 is
    # 0.19.1 bar its `__version__` strings. The lock takes 0.19.2.
    #
    # Previously:
    #
    # Floor was 0.19.0, for `AsyncBusConsumer.claim_stale_page` (cannobserv#465,
    # #109): XAUTOCLAIM's cursor, poison frames returned rather than raised, and
    # the trimmed-while-pending ids. `claim_once` walks the pending list on that
    # cursor, so on an older wheel the worker fails at its first reclaim with an
    # AttributeError — loud, but at runtime, which is why the floor carries it.
    #
    # **Six minors (0.13 -> 0.19), taken on the same evidence as the 0.10 -> 0.13
    # jump below.** Diffing every imported module between the 0.13.2 and 0.19.0
    # wheels: `pure/models/changes.py` moved docstrings only, so no payload field
    # moved under `extra="ignore"`; `pure/adapters/bus/*`, `effects/fetch.py` and
    # `pure/util/hashing.py` are byte-identical. The gcs modules are additive
    # (`GcsCreateIfAbsent.data` also takes a `PurePath`; the md5 moved to a shared
    # `md5_hash` with the same base64 spelling). `co_core_aio.fetch`'s #391 retry
    # policy and #392 deadline apply only to a client the driver builds, and the
    # worker injects its own behind the destination guard (`build_fetch_client`),
    # so neither reaches this service. `claim_stale` is now a wrapper over
    # `claim_stale_page` with unchanged behaviour. The CHANGELOG's Breaking entries
    # across the range are co/v1 WordPress and transcript models, none imported.
    #
    # Previously:
    #
    # Floor was 0.13.1, for `group_name` — the group-naming convention became
    # co-core's to define (cannobserv#384), so `src/core/config.py` derives both
    # command-group defaults rather than spelling them. A *pure no-op at runtime*:
    # both calls evaluate to `replicator.fetch` and `replicator.replicate`, the
    # names already live on the broker (audited 2026-09-01). It buys the same thing
    # the other floors buy — a convention change now arrives with the wheel and
    # trips `test_the_group_defaults_are_derived_not_spelled`, instead of being
    # something someone has to notice. The symptom it forecloses is the quiet one:
    # a renamed convention spelled here as a literal reaches `ensure_group`, which
    # creates a *new empty group* beside the real one, and the worker then reads a
    # stream nothing is delivering to.
    #
    # **This is a three-minor jump (0.10 -> 0.13) and so a departure from "pin the
    # current minor", taken deliberately and on evidence.** The silent-skew hazard
    # the rule guards is payload fields moving under `extra="ignore"`; every module
    # this service imports is byte-identical between 0.10.0 and 0.13.2 —
    # `pure/models/changes.py`, `pure/adapters/bus/{envelope,exceptions}.py`,
    # `effects/{bus,fetch,gcs}.py`, `pure/util/{hashing,gcs}.py`, and the whole of
    # `co_core_aio` bar its `__version__` string. `streams.py` is the only surface
    # that gained anything. Replicator imports no `co_v1` adapter, which is where
    # 0.13.2's other changes landed. Verified by diffing the wheels, not inferred
    # from the version numbers.
    #
    # Ceiling stays at the minor. Floor is 0.13.1 and not 0.13.2 because 0.13.1 is
    # where `group_name` landed and `streams.py` is identical across the two — the
    # floor names the feature depended on, per the rule in docs/DEPLOYMENT.md.
    #
    # Everything below is the history that rule was built from, kept because each
    # entry records a failure mode rather than a version number.
    #
    # Floor was 0.9.4, and every floor since 0.8.0 has been chosen to fail *loudly*
    # rather than silently. The earlier ones (0.7.5's enriched fields, 0.7.7's
    # AsyncBusTailReader) guarded against extra="ignore" quietly discarding kwargs
    # an older wheel had no field for; 0.8.0 makes info_source_id **required** on
    # ContentFetchCommand, BlobAvailableEvent and FetchFailedEvent, so a skew is a
    # ValidationError at construction, not a fact that looks right and carries
    # nothing (#28).
    #
    # 0.10 adds the gcs extra: AsyncGcsDriver.create_if_absent, the T4 primitive
    # (cannobserv#333). The **extra** is the load-bearing part of this line —
    # ``co-core-aio[bus]`` resolves fine and then fails at import, because
    # google-cloud-storage is only pulled in by [gcs]. A floor that admits the
    # right version with the wrong extras is the silent-skew shape this pin
    # exists to prevent, one dimension over from the version itself.
    #
    # 0.9.4 cut the replicate contracts — ContentReplicateCommand,
    # ReplicationCompleteEvent, ReplicationFailedEvent, and the CONTENT_REPLICATE /
    # CONTENT_ARTIFACTS stream constants (#29, cannobserv#303). This floor is the
    # loudest of the three mechanisms: on an older wheel the import fails outright
    # at module load, so a skew cannot reach a running worker at all. Additive over
    # 0.8.0 for everything the fetch path uses — no field on ContentFetchCommand,
    # BlobAvailableEvent or FetchFailedEvent moved between the two.
    "co-core>=0.40.0,<0.41",
    "co-core-aio[bus,gcs]>=0.40.0,<0.41",
    "co-core-sync[gcs]>=0.40.0,<0.41",
    "fastapi>=0.126.0,<1",
    "pydantic>=2.9,<3",
    "pydantic-settings>=2.5.0,<3",
    "python-json-logger>=4.0.0,<5",
    "python-ulid>=3.1.0,<4",
    # Viewport capture's renderer (#138). **Pinned exact, not a range**: each
    # Playwright release pins one Chromium revision, and the worker launches the
    # `chromium-headless-shell` that `playwright install` put under
    # REPLICATOR_PLAYWRIGHT_BROWSERS_PATH for *this* version. A patch bump moves
    # the revision too, so a bump is a lock change plus the runbook's install
    # step on every host (docs/plans/2026-10-25-viewport-capture-runbook.md).
    # Bundles its own Node (the driver): an outside-apt runtime for patching.
    "playwright==1.64.1",
    "uvicorn[standard]>=0.34.0,<1",
]

[project.scripts]
# Canonical launch name. deploy/replicator.service and docs invoke the module
# form (python -m src.worker.main); this gives the entry point a stable name if
# the module ever moves.
replicator-worker = "src.worker.main:main"

[dependency-groups]
dev = [
    "anyio>=4.0,<5",
    "fakeredis>=2.26,<3",
    "httpx>=0.27,<1",
    "pre-commit>=4.0,<5",
    "pytest>=9.0,<10",
    "pytest-asyncio>=1.0,<2",
    "pytest-cov>=7.0,<8",
    "pytest-timeout>=2.3,<3",
    "pyyaml>=6,<7",
    "ruff>=0.16,<0.17",
    "ty",  # beta — deliberately unpinned; non-gating advisory checker
]

# cannobserv workspace packages (co-core, co-core-aio) resolve from the local
# wheelhouse, mirrored from the private GCS index gs://co-gcs-pypi by
# scripts/sync_wheelhouse.py before `uv sync`/`uv run`. Plain version floors
# above + find-links here; no hashes, because find-links locks by filename.
[tool.uv]
find-links = ["./.wheelhouse"]

[tool.pytest.ini_options]
testpaths = ["tests"]
# --strict-markers: an unregistered marker is an error, not a warning. A typo'd
# `@pytest.mark.gsc` would otherwise leave a test that never runs under any
# selector and never reports why (CR #9).
addopts = "-v --tb=short --strict-markers -m 'not integration and not gcs and not browser' --cov=src --cov-report=term-missing"
asyncio_mode = "auto"
asyncio_default_fixture_loop_scope = "session"
asyncio_default_test_loop_scope = "session"
# LEFTOVER_TTL_SECONDS in tests/conftest.py is sized above this — raising the
# timeout past that TTL weakens the scratch-key guarantee. Change one, revisit
# the other.
timeout = 300
timeout_method = "thread"
markers = [
    "integration: marks tests that hit live external services or a real Redis broker; excluded by default (run with -m integration)",
    "browser: marks tests that launch the pinned headless Chromium; excluded by default (run with -m browser). A missing runtime fails them, never skips — see docs/TESTING.md",
    "gcs: marks tests that write to the provisioned GCS test bucket; excluded by default (run with -m gcs). Deliberately not `integration` — see docs/TESTING.md",
]
filterwarnings = [
    "error::DeprecationWarning",
]

[tool.coverage.run]
source = ["src"]

[tool.coverage.report]
show_missing = true
fail_under = 80

[build-system]
requires = ["uv_build>=0.12.0,<0.13"]
build-backend = "uv_build"

[tool.uv.build-backend]
module-name = "src"
module-root = ""

[tool.ruff]
line-length = 100
target-version = "py312"
extend-exclude = ["skills-vendor/"]

[tool.ruff.lint]
select = ["E", "F", "I", "W", "UP", "B904", "PLC0415", "FAST", "ASYNC"]

[tool.ruff.lint.per-file-ignores]
"tests/**" = ["PLC0415"]

[tool.ruff.lint.isort]
known-first-party = ["src"]

# ty walks the whole tree by default. skills-vendor/ is read-only third-party
# skill content — 9 of 10 diagnostics came from there, drowning the one real
# signal. Mirrors ruff's extend-exclude.
[tool.ty.src]
exclude = ["skills-vendor/"]