From 55dd3ece44a1659855dac0b38ef0321444efed8e Mon Sep 17 00:00:00 2001 From: Kevin Ferguson Date: Wed, 28 Oct 2026 21:32:08 +0000 Subject: [PATCH] Add the Street Manager Open Data archive (streetworks.opendata.archive) DfT publishes every Open Data event as public monthly zips per topic (permit from 2020/07, activity from 2020/09, section 58 from 2022/03). Probed live before building; the full record is in docs/findings/streetmanager-opendata-archive.md. - ArchiveClient: url(), available() (HEAD; 404 = not yet published), files() (the host's public S3 bucket listing), and download() - streamed to a .part file, resumed with Range/If-Range, and verified against the ETag, which is the file's MD5. - iter_events(): reads a zip or a folder of zips one member at a time, yielding EventNotification, with filters named after the live feed's SNS attributes (ha_org/promoter_org are SWA codes). Bad zips and members are skipped and counted on ReadStats. No de-duplication: none was found within or across months. - Registered as opendata_archive; documented in uk.md, including the sizes (a permit month is ~1 GB), the ~0.7 GB zipfile memory cost for a permit month, and a warning not to pass archive events to from_streetmanager(), which accepts them but drops dates, geometry, status and PAA classification. - Tests use real fixtures: two pages of the live listing and small zips of real members from each topic's 2026/09 file. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 29 ++ .../streetmanager-opendata-archive.md | 106 +++++ docs/providers/index.md | 3 +- docs/providers/uk.md | 72 ++++ src/streetworks/opendata/archive.py | 368 ++++++++++++++++++ src/streetworks/registry.py | 22 ++ ...anager_archive_activity_2026-09_sample.zip | Bin 0 -> 4651 bytes .../streetmanager_archive_listing_page1.xml | 2 + .../streetmanager_archive_listing_page2.xml | 2 + ...tmanager_archive_permit_2026-09_sample.zip | Bin 0 -> 10505 bytes ...ager_archive_section_58_2026-09_sample.zip | Bin 0 -> 6834 bytes tests/test_opendata_archive.py | 226 +++++++++++ 12 files changed, 829 insertions(+), 1 deletion(-) create mode 100644 docs/findings/streetmanager-opendata-archive.md create mode 100644 src/streetworks/opendata/archive.py create mode 100644 tests/fixtures/streetmanager_archive_activity_2026-09_sample.zip create mode 100644 tests/fixtures/streetmanager_archive_listing_page1.xml create mode 100644 tests/fixtures/streetmanager_archive_listing_page2.xml create mode 100644 tests/fixtures/streetmanager_archive_permit_2026-09_sample.zip create mode 100644 tests/fixtures/streetmanager_archive_section_58_2026-09_sample.zip create mode 100644 tests/test_opendata_archive.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b93c03..e6dc454 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,35 @@ ## [Unreleased] +### Added — Street Manager Open Data archive, every Open Data event as public monthly zips (2026-10-28) + +`streetworks.opendata.archive` +- DfT's archive of the Open Data feed: one zip per topic per month at + `opendata.manage-roadworks.service.gov.uk/{topic}/{yyyy}/{mm}.zip`. Permits from 2020/07, + activities from 2020/09, section 58s from 2022/03, no gaps, no credentials, OGL v3. A permit + month is about 1 GB (about 1.1 million events; about 71 GB in all), so nothing downloads + unless asked. Confirmed live before building - full record in + `docs/findings/streetmanager-opendata-archive.md`. +- `ArchiveClient`: `url()` (no I/O), `available()` (one `HEAD`; a month not yet published is a + 404), `files()` (the host's public S3 bucket listing, so what's published is listed rather + than guessed), and `download()` - streamed to a `.part` file and renamed once complete, + resumed with an HTTP `Range` (+ `If-Range`) request, and verified against the `ETag`, which + is the file's MD5. A resumed `.part` left from an older copy of the file is caught by that + check and the download restarted once. +- `iter_events()` reads a zip, or every zip under a folder, one member at a time, yielding the + existing `EventNotification`. Filters are named after the live feed's SNS message attributes + (`usrn`, `area`, `ha_org`, `promoter_org`, `activity_type`, plus `event_types`); `ha_org` and + `promoter_org` match SWA codes, as the live attributes do. Bad zips and members are skipped + and counted on a `ReadStats`, never raised. No de-duplication: none was found within or + across months. +- Registered as `opendata_archive` (`source_grade="register"`). Documented in + `docs/providers/uk.md` with the sizes, the publication timing (normally the 1st of the next + month; months cut at UK local midnight) and the memory cost: Python's `zipfile` holds about + 0.7 GB for a permit month's member list while reading it. +- Not built yet: folding events into the shared `Works` model, which is a derived view. +- Tests use real fixtures: two pages of the live bucket listing, and small zips of real + members (original names and bytes) from the 2026/09 files of each topic. + ## [0.11.0] - 2026-10-28 ### Removed — `WorkAPI.create_alteration` and `WorkAPI.assess_alteration` (breaking) (2026-10-28) diff --git a/docs/findings/streetmanager-opendata-archive.md b/docs/findings/streetmanager-opendata-archive.md new file mode 100644 index 0000000..6ff0092 --- /dev/null +++ b/docs/findings/streetmanager-opendata-archive.md @@ -0,0 +1,106 @@ +# Street Manager Open Data archive — live probe record (2026-10-28) + +What was checked against `https://opendata.manage-roadworks.service.gov.uk` +before `streetworks.opendata.archive` was built, and what each result decided. +Everything here was observed live on 2026-10-28; nothing is inferred from +documentation alone. + +## The host + +S3 (bucket `prod-sm-archived-notifications-48575862-eu-west-2`) behind +CloudFront. The DfT [Archived notifications](https://department-for-transport-streetmanager.github.io/street-manager-docs/archived-notifications) +page is a small Vue app that reads the bucket listing directly. + +| Probe | Result | Decided | +|---|---|---| +| `HEAD permit/2026/09.zip` | `200`, `Content-Type: application/zip`, `Content-Length: 1044935949`, `Accept-Ranges: bytes`, `ETag`, `Last-Modified` | Downloads stream and can resume | +| `HEAD` on a missing month (`permit/2026/12.zip`, `permit/2019/12.zip`, unknown topic) | `404` (not `403`) | `available()` treats 404 as "not published"; any other error raises | +| `GET /` and `GET /?list-type=2&prefix=...` | Public `ListBucketResult` (V1 and V2, with continuation tokens) | `files()` lists what's published rather than probing URLs | +| `Range: bytes=N-` | `206` | Resume works | +| `Range` + matching `If-Range: ` | `206`; with a stale `If-Range`, `200` and the whole file | Resume is guarded against the file changing mid-download | +| MD5 of the downloaded `activity/2026/09.zip` and `section_58/2026/09.zip` | Equal to their `ETag`s; no `ETag` in the listing has a multipart `-N` suffix | `download()` verifies against the `ETag` | + +## Coverage and publication + +The full listing is 203 keys, all `{topic}/{yyyy}/{mm}.zip`: + +| Topic | Months | Range | Gaps | Total | +|---|---|---|---|---| +| `permit` | 75 | 2020/07–2026/09 | none | 70.98 GB | +| `activity` | 73 | 2020/09–2026/09 | none | 0.48 GB | +| `section_58` | 55 | 2022/03–2026/09 | none | 0.11 GB | + +Each month normally appears on the 1st of the following month, 08:00–09:45 +UTC (2026/07, /08 and /09 all did); 2026/06 appeared on 21 July. All 34 +files for 2020–2021 months carry 2022 `LastModified` dates, so the archive +only began in 2022 and its first years were uploaded then. + +Months are cut at **UK local midnight**: the 2026/08 `section_58` file holds +one event at `2026-07-31T23:19:59Z` (00:19 BST on 1 August). + +## Contents + +- **One event per member, the live feed's envelope.** Every member in the + 2026/09 `activity` (16,531) and `section_58` (3,764) files, and in 183 + members sampled across the 2026/09 `permit` file, is a JSON object with + exactly `event_reference`, `event_type`, `event_time`, `object_type`, + `object_reference`, `version` and `object_data`. No unparseable members, no + folders. +- **Matches DfT's schema.** Every `object_data` key seen is in DfT's + [Event Notifier Message JSON Schema](https://department-for-transport-streetmanager.github.io/street-manager-docs/api-documentation/json/event-notifier-message.json), + and the event types seen are exactly the schema's 18 (11 permit/work, 3 + activity, 4 section 58). +- **Member names** follow + `{object_reference}_{event_type lowercased}_event_{yyyy-mm-dd}_{HH-MM-SS}-{n}.json`. + Two reasons the SDK doesn't parse them: permit references can contain + underscores (e.g. `..._1-01_permit_submitted_event_...`), so splitting is + ambiguous; and the trailing `{n}` isn't milliseconds but the minute and + second run together unpadded (`12-40-33-4033`, `11-15-09-159`). +- **Filter fields are strings.** `usrn`, `highway_authority_swa_code`, + `promoter_swa_code`, `area_name` and `activity_type` are strings in every + topic. Activity and section 58 events have no promoter fields; section 58 + events have no `activity_type`. +- **No personal data fields.** None of the `object_data` keys hold contact + details, so real members can be committed as fixtures. + +## Duplicates + +None found: no repeated `(object_reference, event_type, event_time)` and no +repeated `event_reference` within any of the 2026/08 and 2026/09 `activity` +and `section_58` files, and no overlap between the two months on either key. +The guard in `KFergusonUK/RemedialWorksVisits`'s `remedial_works.py` reads +loose `.json` files alongside zips in the same folder, so it protects against +an extracted member being counted twice - not against anything in DfT's +files. So the SDK doesn't de-duplicate; `event_reference` is unique if a +caller mixes sources. Not checked for `permit` across months (it would need +two 1 GB downloads). + +## The permit file, read over HTTP ranges + +`permit/2026/09.zip` was opened with `zipfile` through a range-backed file +object, fetching about 137 MB rather than 1 GB: + +- **ZIP64**, 1,081,996 members, a 125 MB central directory. +- **Memory:** `zipfile.ZipFile` held 723 MB after opening (865 MB peak), + measured with `tracemalloc`. This is the cost `iter_events()` documents for + a permit month. +- **Every member uses a data descriptor** (general-purpose flag bit 3), all + deflate. A constant-memory reader that walks local headers instead of the + central directory is possible, but it has to find each member's end by + decompressing it, since the local headers carry no sizes. Not built. + +## SNS filter attributes + +The live feed's `MessageAttributes` are `area`, `ha_org`, `activity_type`, +`promoter_org` and `usrn`. DfT's sample notification shows `ha_org` and +`promoter_org` carry SWA codes (`5990`, `STPR`), so `iter_events()`'s filters +of the same names match `highway_authority_swa_code` and `promoter_swa_code`, +not the organisation names. + +## Licence + +The archive host states none. The DfT docs page that hosts the archive browser +carries the site footer "All content is available under the Open Government +Licence v3.0, except where otherwise stated", and nothing on the page states +otherwise. The archive needs no registration, unlike the live SNS feed, whose +sign-up includes accepting terms of use. diff --git a/docs/providers/index.md b/docs/providers/index.md index a6d0719..f5695ba 100644 --- a/docs/providers/index.md +++ b/docs/providers/index.md @@ -70,7 +70,7 @@ Street Manager Credentials: Street Manager API account (email + password) from streetworks.streetmanager import StreetManagerClient -... (5 more — OS Open USRN, DataVIA, D-TRO, Street Manager Open Data, UK Police) +... (8 more — Street Manager Open Data, Street Manager Open Data archive, DataVIA, D-TRO, OS Open USRN, National Highways, TfL, UK Police) >>> DGTClient = get_provider("dgt") # the class, not an instance - constructors vary >>> with DGTClient() as dgt: @@ -115,6 +115,7 @@ client, documented in its own section in [`docs/providers/`](.). |---|---|---| | `streetworks.streetmanager` | [DfT Street Manager](https://department-for-transport-streetmanager.github.io/street-manager-docs/api-documentation/) — all nine APIs (Work, Reporting, Street Lookup, GeoJSON, Party, Data Export, Event, Sampling, Worklist), V6 & V7, sandbox & production | read + write | | `streetworks.opendata` | [Street Manager Open Data](https://department-for-transport-streetmanager.github.io/street-manager-docs/open-data/) — AWS SNS push notifications | receive | +| `streetworks.opendata.archive` | [Street Manager Open Data archive](https://department-for-transport-streetmanager.github.io/street-manager-docs/archived-notifications) — every Open Data event as public monthly zips: permits from 2020/07, activities from 2020/09, section 58s from 2022/03 (no credentials, confirmed live 2026-10-28). A permit month is about 1 GB, about 71 GB in all; nothing downloads unless asked. Licence OGL v3. See [`docs/providers/uk.md`](uk.md#street-manager-open-data-archive) | read | | `streetworks.datavia` | [Geoplace DataVIA](https://datavia.geoplace.co.uk/documentation) — full NSG layer catalogue over OGC WFS and WMS (rendered maps + feature info), Basic + OAuth2 | read | | `streetworks.dtro` | [DfT Digital Traffic Regulation Orders](https://d-tro.dft.gov.uk/api-documentation/) — the legal orders behind speed limits, closures and restrictions; integration & production | read + write | | `streetworks.srwr` | [Scottish Road Works Register](https://roadworks.scot/) — national register via Open Data CSV extracts (no credentials) | read | diff --git a/docs/providers/uk.md b/docs/providers/uk.md index c6dd27d..9206bc6 100644 --- a/docs/providers/uk.md +++ b/docs/providers/uk.md @@ -150,6 +150,78 @@ complete FastAPI receiver. > credentials, though the messages, once flowing, are received the same > credential-free way. This module handles the receiving side of both. +## Street Manager Open Data archive + +DfT also publishes every Open Data event as a public archive, linked from its +[Archived notifications](https://department-for-transport-streetmanager.github.io/street-manager-docs/archived-notifications) +page: one zip per topic per month. No credentials or registration; licence +OGL v3 (the docs site footer on that page). `streetworks.opendata.archive` +lists, downloads and reads it: + +```python +from streetworks.opendata.archive import ArchiveClient, ReadStats, Topic, iter_events + +with ArchiveClient() as arc: + arc.files(Topic.ACTIVITY) # every published month, from the bucket listing + arc.available(Topic.PERMIT, 2026, 10) # one HEAD; False until it's published + arc.download(Topic.PERMIT, 2026, 9, "sm/permit/2026-09.zip") # ~1 GB, streamed to disk + +stats = ReadStats() +for event in iter_events("sm/", event_types={"PERMIT_SUBMITTED"}, ha_org="1355", stats=stats): + print(event.object_reference, event.object_data["activity_type"]) +print(stats) # zips read, members, events yielded, and anything skipped +``` + +Know the sizes before you download. Confirmed live 2026-10-28: + +| Topic | From | Per month | Events per month | +|---|---|---|---| +| `permit` | 2020/07 | 0.85–1.2 GB | about 1.1 million | +| `activity` | 2020/09 | 10–25 MB | about 15,000 | +| `section_58` | 2022/03 | 2–4 MB | about 3,500 | + +The whole permit archive is about 71 GB. Nothing downloads unless you call +`download()`, one month at a time. Each month normally appears on the 1st of +the following month, and months are cut at UK local midnight, not UTC. + +- **Downloads resume and are verified.** Bytes go to a `.part` file that is + renamed only once complete; running `download()` again continues it with an + HTTP range request. The finished file is checked against the server's + `ETag`, which is the file's MD5. +- **Each event is the live feed's message**, as a + `streetworks.opendata.EventNotification` - `event_type`, `event_time`, + `object_reference` and `object_data`, the same fields DfT's Event Notifier + Message schema documents. +- **Filters are named after the live feed's SNS message attributes**: `usrn`, + `area`, `ha_org` and `promoter_org` (SWA codes, not organisation names) and + `activity_type`, plus `event_types`. Each takes one value or a collection. +- **Bad files are skipped, not fatal.** A zip that won't open or a member + that isn't an event is counted on `ReadStats` and skipped. +- **Nothing is de-duplicated.** No duplicate events were found within a month + or across neighbouring months; `event_reference` is unique if you combine + overlapping sources yourself. +- **Memory.** Reading a permit month holds about 0.7 GB (peaking near 0.9 GB) + for Python's `zipfile` member list until that zip is done. Activity and + section 58 months are small. + +The archive is events, not works: one permit appears as several events +(submitted, granted, work start, work stop...). Turning those into the shared +`Works` model means folding events per reference, which this SDK doesn't do +for you yet. + +> **Don't pass archive events to `from_streetmanager()`.** A permit event's +> `object_data` looks like a Reporting permit row, and the converter accepts +> it without error, but the field names differ and the results are wrong: +> actual start/end dates are dropped (the archive calls them +> `actual_start_date_time`/`actual_end_date_time`) and the dates graded +> "estimated"; the geometry is dropped (the archive's +> `works_location_coordinates` is WKT text in British National Grid, not +> GeoJSON); status and works type come out empty; and PAAs become sites +> instead of plannings (the archive's `work_category` reads `"Major (PAA)"`; +> the code `paa` is in `work_category_ref`). Each event would also become its +> own record, so one permit appears several times. Read `object_data` +> directly until the archive has a converter of its own. + ## Transport for London (TfL Road Disruption) London roadworks are already in this SDK via Street Manager (the diff --git a/src/streetworks/opendata/archive.py b/src/streetworks/opendata/archive.py new file mode 100644 index 0000000..86bc2fd --- /dev/null +++ b/src/streetworks/opendata/archive.py @@ -0,0 +1,368 @@ +"""Street Manager Open Data archive - every Open Data event, as monthly zips. + +DfT publishes the events behind the Street Manager Open Data feed (the same +messages the SNS feed pushes - see :mod:`streetworks.opendata`) as a public +archive: one zip per topic per month, at +``https://opendata.manage-roadworks.service.gov.uk/{topic}/{yyyy}/{mm}.zip``, +linked from DfT's "Archived notifications" page. No credentials or +registration. Licence: Open Government Licence v3.0, per the footer of the +DfT docs page that hosts the archive browser (the archive host itself states +none). + +Confirmed live 2026-10-28, from the bucket listing, HEAD/GET requests and real +files: + +* **Three topics**, no gaps: ``permit`` from 2020/07, ``activity`` from + 2020/09, ``section_58`` from 2022/03. +* **Published monthly**, normally on the 1st of the following month (June + 2026 came on 21 July). Months are cut at UK local midnight, not UTC: an + event at ``2026-07-31T23:19Z`` (00:19 BST) is in the August file. +* **Sizes.** ``permit`` is about 0.85-1.2 GB a month (about 1.1 million + events) and about 71 GB for the whole archive; ``activity`` is 10-25 MB a + month; ``section_58`` 2-4 MB. +* **One event per zip member**, named + ``{object_reference}_{event_type}_event_{date}_{time}-{n}.json``; each is + the envelope :class:`~streetworks.opendata.EventNotification` models, and + every ``object_data`` field seen matches DfT's Event Notifier Message + schema. Names aren't parsed here: permit references can contain + underscores, so the JSON is the only reliable source. +* **No duplicates seen** - none within a month, and none across Aug/Sep 2026 + for ``activity`` and ``section_58`` (by ``event_reference``, or by + reference, event type and time). So nothing is de-duplicated. +* **The host is S3 behind CloudFront**: a public bucket listing (so + :meth:`ArchiveClient.files` lists what's published rather than guessing), + 404 for a month not yet published, ``Accept-Ranges: bytes`` (so + :meth:`ArchiveClient.download` resumes), and an ``ETag`` that is the file's + MD5 (so downloads are verified). + +Nothing here downloads anything unless asked: :meth:`ArchiveClient.download` +streams one file to disk, and :func:`iter_events` reads zips already on disk. +""" + +from __future__ import annotations + +import hashlib +import json +import re +import zipfile +import zlib +from collections.abc import Iterable, Iterator +from dataclasses import dataclass +from enum import Enum +from pathlib import Path +from xml.etree import ElementTree + +import httpx +from pydantic import ValidationError + +from .._transport import RetryConfig, SyncTransport, _raise_for_response +from ..exceptions import NotFoundError, TransportError +from .models import EventNotification + +__all__ = [ + "BASE_URL", + "ArchiveClient", + "ArchiveFile", + "ReadStats", + "Topic", + "iter_events", +] + +BASE_URL = "https://opendata.manage-roadworks.service.gov.uk" + +_S3 = {"s3": "http://s3.amazonaws.com/doc/2006-03-01/"} +_KEY = re.compile(r"^(?P[a-z0-9_]+)/(?P\d{4})/(?P\d{2})\.zip$") +_MD5_ETAG = re.compile(r'^"?(?P[0-9a-f]{32})"?$') + + +class Topic(str, Enum): + PERMIT = "permit" + ACTIVITY = "activity" + SECTION_58 = "section_58" + + +_TOPICS = frozenset(t.value for t in Topic) + + +@dataclass(frozen=True) +class ArchiveFile: + """One published month, as the bucket listing describes it.""" + + topic: Topic + year: int + month: int + url: str + size: int + last_modified: str + etag: str + + +class ArchiveClient: + """List and download the Street Manager Open Data archive. No + credentials required. + + >>> from streetworks.opendata.archive import ArchiveClient, Topic + >>> with ArchiveClient() as arc: + ... arc.available(Topic.PERMIT, 2026, 9) + ... arc.download(Topic.PERMIT, 2026, 9, "sm/permit-2026-09.zip") # ~1 GB + """ + + def __init__( + self, + *, + base_url: str = BASE_URL, + retry: RetryConfig | None = None, + timeout: float = 600.0, + client: httpx.Client | None = None, + ): + self.base_url = base_url.rstrip("/") + self._client = client or httpx.Client(timeout=timeout, follow_redirects=True) + self._transport = SyncTransport( + retry=retry or RetryConfig(), timeout=timeout, client=self._client + ) + + def url(self, topic: Topic | str, year: int, month: int) -> str: + """The URL a month's zip is (or will be) published at. No I/O.""" + if not 1 <= month <= 12: + raise ValueError(f"month must be 1-12, got {month}") + return f"{self.base_url}/{Topic(topic).value}/{year:04d}/{month:02d}.zip" + + def available(self, topic: Topic | str, year: int, month: int) -> bool: + """Whether a month has been published yet (one ``HEAD``; a month not + yet published is a 404).""" + try: + self._transport.request("HEAD", self.url(topic, year, month)) + except NotFoundError: + return False + return True + + def files(self, topic: Topic | str | None = None) -> list[ArchiveFile]: + """Every published month, from the public bucket listing, by topic + and oldest first - optionally for one topic. Keys that aren't a + ``{topic}/{yyyy}/{mm}.zip`` for a known topic are left out.""" + params = {"list-type": "2"} + if topic is not None: + params["prefix"] = f"{Topic(topic).value}/" + found: list[ArchiveFile] = [] + while True: + root = ElementTree.fromstring( + self._transport.request("GET", f"{self.base_url}/", params=params).content + ) + for entry in root.findall("s3:Contents", _S3): + key = entry.findtext("s3:Key", "", _S3) + match = _KEY.match(key) + if not match or match["topic"] not in _TOPICS: + continue + found.append( + ArchiveFile( + topic=Topic(match["topic"]), + year=int(match["year"]), + month=int(match["month"]), + url=f"{self.base_url}/{key}", + size=int(entry.findtext("s3:Size", "0", _S3)), + last_modified=entry.findtext("s3:LastModified", "", _S3), + etag=entry.findtext("s3:ETag", "", _S3), + ) + ) + token = root.findtext("s3:NextContinuationToken", None, _S3) + if root.findtext("s3:IsTruncated", "false", _S3) != "true" or not token: + break + params["continuation-token"] = token + return sorted(found, key=lambda f: (f.topic.value, f.year, f.month)) + + def download( + self, + topic: Topic | str, + year: int, + month: int, + dest: str | Path, + *, + resume: bool = True, + verify: bool = True, + ) -> Path: + """Stream one month's zip to ``dest`` and return its path. + + A permit month is about 1 GB; it is streamed in 1 MB chunks, never + held in memory. Bytes go to ``dest`` + ``.part`` and are renamed into + place only once complete, so an interrupted run never leaves a + truncated file that looks like a good zip. With ``resume``, a + leftover ``.part`` is continued with an HTTP ``Range`` request. With + ``verify``, the finished file's MD5 is checked against the ``ETag``; + if a resumed ``.part`` turns out to belong to an older copy of the + file, it is discarded and the download restarted once from scratch. + Raises :class:`~streetworks.exceptions.NotFoundError` for a month not + yet published, and :class:`~streetworks.exceptions.TransportError` if + the file doesn't match its ``ETag``. + """ + url = self.url(topic, year, month) + dest = Path(dest) + dest.parent.mkdir(parents=True, exist_ok=True) + part = dest.with_name(dest.name + ".part") + head = self._transport.request("HEAD", url) + size = int(head.headers["content-length"]) + etag = head.headers.get("etag", "") + md5 = _MD5_ETAG.match(etag) + + for attempt in (1, 2): + offset = part.stat().st_size if resume and part.exists() else 0 + if offset > size or (offset and not etag): + offset = 0 + if offset < size: + headers = {"Range": f"bytes={offset}-", "If-Range": etag} if offset else {} + with self._client.stream("GET", url, headers=headers) as response: + if response.status_code >= 400: + response.read() + _raise_for_response(response) + if response.status_code != 206: + offset = 0 # whole file sent: it changed, or the range was ignored + with open(part, "ab" if offset else "wb") as f: + for chunk in response.iter_bytes(chunk_size=1024 * 1024): + f.write(chunk) + if part.stat().st_size != size: + raise TransportError( + f"{url} downloaded {part.stat().st_size} of {size} bytes; " + f"run again to resume from {part}" + ) + if not verify or md5 is None or _md5(part) == md5["md5"]: + part.replace(dest) + return dest + part.unlink() + if not offset or attempt == 2: + raise TransportError(f"{url} does not match its ETag {etag} after download") + raise AssertionError("unreachable") # pragma: no cover + + def close(self) -> None: + self._transport.close() + + def __enter__(self) -> ArchiveClient: + return self + + def __exit__(self, *exc_info: object) -> None: + self.close() + + +def _md5(path: Path) -> str: + digest = hashlib.md5(usedforsecurity=False) + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +@dataclass +class ReadStats: + """Counts from one :func:`iter_events` pass - pass one in to see what was + read and what was skipped.""" + + zips: int = 0 + bad_zips: int = 0 + members: int = 0 + bad_members: int = 0 + events: int = 0 + + +# iter_events filter -> the object_data field it matches. The names are the +# SNS MessageAttributes the live feed can be filtered on; ha_org and +# promoter_org carry SWA codes, not organisation names. +_FILTER_FIELDS = { + "usrn": "usrn", + "area": "area_name", + "ha_org": "highway_authority_swa_code", + "promoter_org": "promoter_swa_code", + "activity_type": "activity_type", +} + +FilterValue = str | int | Iterable[str | int] | None + + +def _as_set(value: FilterValue) -> frozenset[str] | None: + if value is None: + return None + if isinstance(value, (str, int)): + return frozenset({str(value)}) + return frozenset(str(v) for v in value) + + +def iter_events( + source: str | Path, + *, + event_types: str | Iterable[str] | None = None, + usrn: FilterValue = None, + area: FilterValue = None, + ha_org: FilterValue = None, + promoter_org: FilterValue = None, + activity_type: FilterValue = None, + stats: ReadStats | None = None, +) -> Iterator[EventNotification]: + """Yield the events in an archive zip, or in every zip under a folder + (searched recursively, in name order), one member at a time. + + Every filter takes one value or a collection, and keeps an event whose + value matches any of them; filters combine with AND. ``event_types`` + matches ``event_type`` (e.g. ``"PERMIT_SUBMITTED"``); the others match + ``object_data`` and are named after the live feed's SNS + MessageAttributes: ``usrn``, ``area`` (``area_name``), ``ha_org`` + (``highway_authority_swa_code``), ``promoter_org`` + (``promoter_swa_code``) and ``activity_type``. An event without the + field doesn't match - activity and section 58 events carry no promoter, + and section 58 events no ``activity_type``. + + A zip that won't open, or a member that won't decompress or isn't an + event, is skipped and counted on ``stats`` - never raised. + + Memory: Python's ``zipfile`` reads a zip's whole member list on open. + For a permit month (about 1.1 million members) that list holds about + 0.7 GB, peaking near 0.9 GB while it is read, until that zip is + finished. Activity and section 58 months are small. + """ + stats = stats if stats is not None else ReadStats() + wanted_types = _as_set(event_types) + filters = { + "usrn": usrn, + "area": area, + "ha_org": ha_org, + "promoter_org": promoter_org, + "activity_type": activity_type, + } + wanted = { + _FILTER_FIELDS[name]: values + for name, value in filters.items() + if (values := _as_set(value)) is not None + } + source = Path(source) + zips = sorted(source.rglob("*.zip")) if source.is_dir() else [source] + for path in zips: + try: + archive = zipfile.ZipFile(path) + except (zipfile.BadZipFile, OSError): + stats.bad_zips += 1 + continue + stats.zips += 1 + with archive: + for info in archive.infolist(): + if info.is_dir(): + continue + stats.members += 1 + try: + raw = json.loads(archive.read(info)) + except (zipfile.BadZipFile, zlib.error, NotImplementedError, ValueError, OSError): + stats.bad_members += 1 + continue + if not isinstance(raw, dict): + stats.bad_members += 1 + continue + if wanted_types is not None and raw.get("event_type") not in wanted_types: + continue + data = raw.get("object_data") or {} + if any( + field not in data or str(data[field]) not in values + for field, values in wanted.items() + ): + continue + try: + event = EventNotification.model_validate(raw) + except ValidationError: + stats.bad_members += 1 + continue + stats.events += 1 + yield event diff --git a/src/streetworks/registry.py b/src/streetworks/registry.py index 414e0aa..6793a4b 100644 --- a/src/streetworks/registry.py +++ b/src/streetworks/registry.py @@ -509,6 +509,28 @@ def get_provider(key: str) -> Any: _client_name="handle", import_line="from streetworks.opendata import handle, parse_message", ), + ProviderEntry( + key="opendata_archive", + name="Street Manager Open Data archive", + description=( + "Every Street Manager Open Data event (permits since 2020, " + "activities, section 58s) as public monthly zips." + ), + kind=Kind.ROADWORKS, + network_scope=NetworkScope.COMPREHENSIVE, + territories=frozenset({"England"}), + scope_note=( + "Historical, not live: one zip per topic per month, normally " + "published on the 1st of the next month. A permit month is about " + "1 GB (about 71 GB since 2020/07); activity and section 58 months " + "are a few MB. Nothing downloads unless asked." + ), + licence="Open Government Licence v3.0 (OGL v3)", + source_grade="register", + _module="streetworks.opendata.archive", + _client_name="ArchiveClient", + import_line="from streetworks.opendata.archive import ArchiveClient, iter_events", + ), ProviderEntry( key="datavia", name="Geoplace DataVIA", diff --git a/tests/fixtures/streetmanager_archive_activity_2026-09_sample.zip b/tests/fixtures/streetmanager_archive_activity_2026-09_sample.zip new file mode 100644 index 0000000000000000000000000000000000000000..3b6fcde69a243a2630f56ea23a6b00a6ea265376 GIT binary patch literal 4651 zcmb`~XHXOBwgzB&7f_0bQbLggh$JBd5Cp-1p(xSX)(k>2d0NRbk% zAOWR{AWeGjRYB>(iF@C>-J5gnoFCp~=1ZAO-apU0YkgOAXz94AsHm8z46`jP%N4Iy zA!(?nj?qz3At~Rg8C(WKp>Qw+0!JvwBNVM|?Qt%CF1YK~zFt>taSm6l9sC?TaMmyg z3;~8Hf}wKOP$=d6^S$C3S07IgcSB3hClJm(siX%hPsNVSOugu_KS#bOP2cI)KoxsI-ql5MX1R)s5P8#+2uI` zKdCr4+5286a}Ucx5e+`NGl*Zm@!BbRwD!Ci3ewkgXCyxf^VMCDZhR=S5W>dCI&)^+ zs!3VJM|~ye(d^Es-U)VXkBE;<*EoW%fzx`wO&Q>G&ElnA+!(@-F!i78p3HjZbC57G z_6i1GXrFb6WYa_Rj$lJa#_?KSWj_AmGx8m9lG3vSvq{l3kU1U3_H*>qq}uN3d;+;d+njbs zAR#dad~S9{v1$BXyh4>ozod*o{JX{QToZY%n#(Ljro#?V({dpp4{D*YW!t=xfrRkNUVo=aF@N zsr@yF|Dcre3kp#I|#^nFXcK?rxmfi^B50m@ryaMxNvQq7KoHCVmc4^D9wF_wkS9bNsB238Z31o2494#mN2V(GSO(KB9-4R%S2Ai zyq%fd!$Yt+f*>I;O4)CY)T=6MN<AmMoNSPPEZ!tNqD*%r!Hj^b155rR1nr7aDvR|0P5EUVTo z3Jhes1@+yDK27_P(ZYx*u7_>n+x`C7CnxAh&_u=@N?z#U^&VLr&w&lGon92KWo~lw z4AEO@s6Hq@BG9F$dh`N~xRy06%HLnRSHRUme{(3}hJ1?wE%Ve&NHbYC-Rv=m#iXP@ zB(G$sZ=~x&+;jJ@y{0%I7y)7yv<_T&;mmJFxGbdFe8=zxyeVLPCS9%UqDhPE0 zHJHlTjG&csHF{T;2t}l$QnrK%n$rqwobS1~11_`)3exM^oXvAob?PYyG`s4OXnBES zVOognJ`8EKS))5qc1>b=JQw{cw2Hf-Fe;c(sH&bq$0ejcN3M}&Io%O!M*zhcwDeMk7%i3ghAdYdQsE#^dS4`98dGI zla5yCSUWAFG0G>z8>dfl)gbvSuI?VizP^!Az!B+_X_?AHXqcfNSN3*unM6*og z8$AOn{nVj!F385{V4UaivFm3a1`RfAhINK3%5n$*;n*;vF)M_HXEKeE-7b*L&?4*z z*4!)c*x2Z^GdA^Tspdtnp&Pig@nEMcSrZPqS+UOt>ocyL`M}rQYuCFm$V z`AsNLjUu;rWmFOg~7m32<3!A{%2sfdJIwq zHjkYiNciN_7n^O~-3{j@kHZXwtC`p|l-wL(&wwX2<8H{zHQi00(zJ2(b8@QC6ZSW# zYG@&q7_c(aYXQZzGN#FQD(asQXdHb?w<>J>+m>TroscRJrtXZc(*hY z`UG8V-jsw}jg7yv6*~6}fXZp3rgNHTd4a0;OX<;}=IBDk(ORrtF9BL=oTk)&fm%K) z$D%M$KcwoROF}k|e}XUuBpM>)fw4{S77zeV#&}&tb}7e9is`J+H9hl>Q=z&oucc703_Th zsV%P!h=lIbOJo_=97Gu`Y^Vy5xiUgKuJ=|f^yoU#`M1p7pDk!8Kc`yjlz`r-%eiNJDo(oT{E>Tf*+78S^7Eh;nPHjw^>{CW7nEOgL∨Q#R?oOxN+L?L(t^H%bC~i2{<9vOsG7tuo~m5Dgwy?R`zqD+93R#_K^FSg8Ufeo&twdM{I}*z<{WpW-CM1%(yG^yfp#hS z3wXv$?s8(BF?K{u7#Tl+0a-rUE)`I;{2*jl^5M8k8lG`TJ5+L*xL)<3xLN*7ZEE$( zK^SB2mmO&|O4#|X1g4J9T(T`Q7jSbIlXPbNy>z-2y{NXGGF$Meu{b2#QB5tr zBN*faXQT8+Ao7C<@5Q5{0^yy2BlEzG`V^}N91F(r`a~D*Z8a1~rC?JjkKQ^@*I2T8 z-vX>XU!LQN!WkMbJC-l=4zC3-6;qRGPtAnJ*<{!#zvDXy=lb~4Z~q?1cUv`<4brUC zd7?lgvv}mak)~HRQN?B7jySkZYZ0DM#EY&j8I7x5oAhmh=cAt?*K?4D9+C;Z{x_K} zm^gQu;stZB-|H3VkV`WwzVF;WOj9}Tf$wPRySOxt#gyA(O;S)OFo8EZ&zvO4WN5tC|5Gb3MQcg^Hv#(3>S$y639a5SvSyyV zJL^b!bm21Z!Z{fNNNH3nfVtBe#P3uFOqW?xN{C5TH2e5AJIW#LN%P)CMCx$;V9>X- z7h+1s1I8{Yl9}1L!f%W6l1jf*?M{wUE~hPIS3&-p{@Q^>&6uv78pCq0c@ z5Ml{v>Ba7lyD?KtR`}se0nVJAQ3+XRms=%zrMT5|qt)v{3CLXsA0gNzy?yCtjAvIgx8w?RNpsRIhhKmcu_+Ar+HHs@L5G0RG192=s!X2gZ2dHgyUkvw zob=;#i*U*xkQu=$*#g!-tR`I@DmCX3Y`qNCMko2$iRN64z24SUS7VRhRehLsSA_FDgBUu2q_*ej;WaioK8C8YOHYE~@?eP*J@}vH& zS+agXwEuHW3e&b^(c-!!zw)lGbp8g8op`-)95_s1iQ`sd64 z$A$ZF;^FJkpLtYNVbsql@c&Bujmy*F#KVj8uS9nGzbF328htqO@Fw&t@-^e%BY$Hr zIvja;*8Pf9WBC>Ndz0^Q=Ha^fmAS_DznQ;ZURQMJ8GimV1LavlF=ET{^V7coF)YeJ literal 0 HcmV?d00001 diff --git a/tests/fixtures/streetmanager_archive_listing_page1.xml b/tests/fixtures/streetmanager_archive_listing_page1.xml new file mode 100644 index 0000000..5507759 --- /dev/null +++ b/tests/fixtures/streetmanager_archive_listing_page1.xml @@ -0,0 +1,2 @@ + +prod-sm-archived-notifications-48575862-eu-west-2section_58/1i9EeqEmy3IOJzovtZWYl+dG0hhEraV6Z9yX0Kv3L8rUmEb0taDkKHvLPgkpuv456+UYoaPYkkpe1V9GGPzeoSDpmuuJZt+CUxWKOupa3lK5W3NA0vkJc1A==22truesection_58/2022/03.zip2022-08-18T14:36:44.000Z"5f611da6a6da3741aa36ebbf6d682303"64360STANDARDsection_58/2022/04.zip2022-08-19T11:14:05.000Z"9954fc01153549a74741508e0c764d1f"395976STANDARD \ No newline at end of file diff --git a/tests/fixtures/streetmanager_archive_listing_page2.xml b/tests/fixtures/streetmanager_archive_listing_page2.xml new file mode 100644 index 0000000..894b0ae --- /dev/null +++ b/tests/fixtures/streetmanager_archive_listing_page2.xml @@ -0,0 +1,2 @@ + +prod-sm-archived-notifications-48575862-eu-west-2section_58/1i9EeqEmy3IOJzovtZWYl+dG0hhEraV6Z9yX0Kv3L8rUmEb0taDkKHvLPgkpuv456+UYoaPYkkpe1V9GGPzeoSDpmuuJZt+CUxWKOupa3lK5W3NA0vkJc1A==531000falsesection_58/2022/05.zip2022-08-19T13:40:31.000Z"635d2e164704fa0d71112ba96361fe53"795942STANDARDsection_58/2022/06.zip2022-08-22T08:26:50.000Z"d69105a1e67e704908973f0f25856e4a"955432STANDARDsection_58/2022/07.zip2022-08-22T12:38:22.000Z"11a02f63194a28b8e62f0b8fffa0fa72"855110STANDARDsection_58/2022/08.zip2022-09-01T14:37:24.000Z"38a4b8bde234c1d584a6313c1696446e"638085STANDARDsection_58/2022/09.zip2022-10-01T08:00:26.000Z"a02a0ac9e1ac5c9315985d4ad1c88ef7"905908STANDARDsection_58/2022/10.zip2022-11-01T08:00:14.000Z"04f39cdbdef8e3e91c08f89a3db39866"979529STANDARDsection_58/2022/11.zip2022-12-13T13:52:47.000Z"3c16e52f02952d48e35cbf29867b202f"1276461STANDARDsection_58/2022/12.zip2023-01-01T08:00:28.000Z"0733220a659f2c19f56e9ce8619ed34d"1003342STANDARDsection_58/2023/01.zip2023-02-01T08:00:19.000Z"1b7199bf98ab028c5cd3aef56f1ff2ad"1732880STANDARDsection_58/2023/02.zip2023-03-01T08:00:18.000Z"fdee666c322fb7ee0e289c83d4f3e2bc"1668259STANDARDsection_58/2023/03.zip2023-04-01T08:00:17.000Z"6b2ba7f6f95fb90b67dd0fa46980c9d3"1516461STANDARDsection_58/2023/04.zip2023-05-01T08:00:22.000Z"85cdebf5bd98313f6d3a81b262f7ff18"1722568STANDARDsection_58/2023/05.zip2023-06-01T08:02:51.000Z"7237d749b61ba3ebfffcd5f56f717ed8"1674571STANDARDsection_58/2023/06.zip2023-07-01T08:00:26.000Z"a5762ac05a9eff7e94f95771b6301439"1409692STANDARDsection_58/2023/07.zip2023-08-01T08:18:52.000Z"6226dabb1db103169ccb9dde8d5e19f7"1624383STANDARDsection_58/2023/08.zip2023-09-01T08:02:41.000Z"64b86a573824e48b17b64bf7263056bb"1835008STANDARDsection_58/2023/09.zip2023-10-01T08:26:41.000Z"14deeaf0a5d1bd35574eb69ddbd4fb9d"1540715STANDARDsection_58/2023/10.zip2023-11-01T08:00:18.000Z"34d500d77185e37b42effbc12bdd8976"1433044STANDARDsection_58/2023/11.zip2023-12-01T08:24:21.000Z"d3f8bda1ef9ee4f8799dbcd382600973"1712128STANDARDsection_58/2023/12.zip2024-01-01T08:00:17.000Z"a83642bba01d954adf694d3f16a1a9a2"1213710STANDARDsection_58/2024/01.zip2024-02-08T17:13:50.000Z"7a7d1bfd3d85496b05ea7b6177fd5146"1736704STANDARDsection_58/2024/02.zip2024-03-01T08:00:25.000Z"d272d00d585d0165af93e8955d52e1e4"1952791STANDARDsection_58/2024/03.zip2024-04-01T08:27:58.000Z"4f973ab6e678295b6122b4c3ef258d9b"2514303STANDARDsection_58/2024/04.zip2024-05-01T08:00:36.000Z"20133dd04befc8257eb36d7693a261b6"2009279STANDARDsection_58/2024/05.zip2024-06-01T08:15:24.000Z"be1759422a49d25b5e2f415d342e27f0"1748769STANDARDsection_58/2024/06.zip2024-07-01T08:00:29.000Z"ef4506d26bb12f42454f8b67ba454a9d"1382523STANDARDsection_58/2024/07.zip2024-08-01T08:04:12.000Z"280cdb781465bcf62231d81408017f11"2441216STANDARDsection_58/2024/08.zip2024-09-01T08:02:59.000Z"320a7c1b434c3fa85b38e8975e81b684"1876273STANDARDsection_58/2024/09.zip2024-10-01T08:00:21.000Z"2358b54561754d4123a24dae7092f1c0"1897912STANDARDsection_58/2024/10.zip2024-11-01T08:02:24.000Z"ab6306c6eac4f5d7fc586cae4aa4af1e"1923747STANDARDsection_58/2024/11.zip2024-12-01T08:02:22.000Z"6b57e8b312b123dfc4daab70da00a7f8"1925120STANDARDsection_58/2024/12.zip2025-01-01T08:00:18.000Z"667cc9b895aa58bb007918bac2f71106"1382907STANDARDsection_58/2025/01.zip2025-02-01T08:17:16.000Z"8e8250621b6f48ee46a6949a535a1708"2289664STANDARDsection_58/2025/02.zip2025-03-01T08:00:21.000Z"12d626fc5248de72960e45181efd21a4"CRC64NVMEFULL_OBJECT1907905STANDARDsection_58/2025/03.zip2025-04-01T08:00:32.000Z"b9eb4eea4d67ab79c82e35ba2e5676fc"CRC64NVMEFULL_OBJECT2392286STANDARDsection_58/2025/04.zip2025-05-01T08:02:28.000Z"ca221bea3491b855e4bde6bebbbcdb94"CRC64NVMEFULL_OBJECT2542347STANDARDsection_58/2025/05.zip2025-06-01T08:00:22.000Z"4ad01d99d8a390ff500c132587e0de58"CRC64NVMEFULL_OBJECT2630544STANDARDsection_58/2025/06.zip2025-07-01T09:33:03.000Z"f413c21707da5124709377a48d95559b"CRC64NVMEFULL_OBJECT2150400STANDARDsection_58/2025/07.zip2025-08-01T08:02:24.000Z"7f9540f8e23b2eb6bec4c8ab7f10d10f"CRC64NVMEFULL_OBJECT2527232STANDARDsection_58/2025/08.zip2025-09-01T08:00:31.000Z"3db52d6816712c60f50e7c663ecf16be"CRC64NVMEFULL_OBJECT2306048STANDARDsection_58/2025/09.zip2025-10-01T08:01:06.000Z"578e867db0f32578076395c8a0370127"CRC64NVMEFULL_OBJECT2264258STANDARDsection_58/2025/10.zip2025-11-01T08:00:25.000Z"5ee1d66cc179b147c36a6697f4a9defe"CRC64NVMEFULL_OBJECT2784146STANDARDsection_58/2025/11.zip2025-12-01T08:18:01.000Z"7c8d25dc0e925d98fe3934103d8f245d"CRC64NVMEFULL_OBJECT2574935STANDARDsection_58/2025/12.zip2026-01-01T08:10:31.000Z"618a8cf1a1be799456b4ffcf73c8c84e"CRC64NVMEFULL_OBJECT2815065STANDARDsection_58/2026/01.zip2026-02-01T10:08:02.000Z"3750a2624eb3142503b40da3f6288319"CRC64NVMEFULL_OBJECT2650695STANDARDsection_58/2026/02.zip2026-03-01T08:01:35.000Z"8b1c7aa834a2fe3e5053671defeeef8f"CRC64NVMEFULL_OBJECT2777875STANDARDsection_58/2026/03.zip2026-04-01T08:03:21.000Z"68c2a81a039fc3c89de83b26dc0d4ff8"CRC64NVMEFULL_OBJECT3491155STANDARDsection_58/2026/04.zip2026-05-01T08:00:35.000Z"ec9fb6bcf0c1c6500dd9743bfe90a36c"CRC64NVMEFULL_OBJECT3222340STANDARDsection_58/2026/05.zip2026-06-01T08:03:12.000Z"3cb420da3493b12a0fa922835165789e"CRC32FULL_OBJECT3417353STANDARDsection_58/2026/06.zip2026-07-21T07:19:37.000Z"3222b413f876160dacda5cda3334d211"CRC64NVMEFULL_OBJECT2268076STANDARDsection_58/2026/07.zip2026-08-01T08:00:35.000Z"c3fb28087836899d1efe039f0459c0b5"CRC32FULL_OBJECT4180984STANDARDsection_58/2026/08.zip2026-09-01T08:02:59.000Z"e93cc6cec8ddd2b440d2d266cd2b2165"CRC32FULL_OBJECT2695675STANDARDsection_58/2026/09.zip2026-10-01T08:00:36.000Z"5427bb520282cd020dbd5891d984e07b"CRC32FULL_OBJECT3089473STANDARD \ No newline at end of file diff --git a/tests/fixtures/streetmanager_archive_permit_2026-09_sample.zip b/tests/fixtures/streetmanager_archive_permit_2026-09_sample.zip new file mode 100644 index 0000000000000000000000000000000000000000..cd8395df0136762e63998afa22e301af7db577f6 GIT binary patch literal 10505 zcmcI~WmKEn+AYQ1iaV4RF9{?;OL2F1N|E61TAbpR;!>=|-KC{C6ff@9Leb*RP50jC z+hcR?_vfB-lZ=&-jCYK=emrwM^O1#?pv9z)1dJApu4f(BsF}<%Orc~qRy)c_5n63zBiPH{j_Nby& zo3tTzX|M3Y=b%i>y`=hJLt@`B=BCR7YE&D4hRnjQwV9{ob3{{Z>)9~SU{?R=lYO;W z9f_~mvswsDec?40OR1J;JqJ(8%~>oU+=MGzQiLcZa+cx9LOubI^{m52B+-H70#~Hp zMG=uOgz-HTIW@E*9B)O;8YkB*f~UM+(EIR?{fN?D&#t-@nYK~nj>1*EjotnBL=)Z4 z@k8{^rXs8DXEf`J;~JvU)rf;i^4WB2B?lA6<<}?gxX4ZNmobswt<5`8`5J zLh;85^3uwvHA!(vaV+#32M3Q!T~5@srt|QNf40bu#j^2TG~$xhub+mGd5=vCEOmKV z>JHlepz?Nb^9p1OaE&Iu7_IBUkr!lgp?P*N)Z?-Hg^~V#dVYvIfAmKG_)!6fa(lR^ zIW`4%1*r7ARFD@E6p&q3TJXXX^<1AKwq(G%QJn-PcXZW+LTfr$3Qi-xsYpaES-LuI z8b#9qrU8QX3`@?UP4iM|pWo!yy*Qe1jSkU-xu7%Iw6kMuwom(KUvU;t`2f zRfA+XiZyjT9e4!SJ~xWml&gpBJg14#sJhx(&?CK6h!4UOod)}lhqfeq1v5urZ}p9h=c|#q<2R4t^{pGZMWTzVP@6cq3r7aN*=Xa9=Y}Gpt{=G z1RrRvRy%{(E#t-|D7q#95sSk(6<6uddoRH&wVRuGp%u>S|FOW%U??IMcN}1rFXj{%f9Z* zi?;~;akcsr)RqZPMQ9XM-BP0`KqvNRjZA&3ZgqYmJl!@1!su4lFxf%2TJ%f2t)x4* z?<>`dwlZS4HPh^}43ME76~R#avwZh2LXoYnUTBwOR_8Zu$0Zj#F;s_FttO6xWsOHM zpHdGV?VV2AkS`2JbW?~nlAij8NyU`4Y3a^9*{GbcA;}KB{JCCzu|RuVVvT-Hgl(@r z>- z2>~O6k!21@_iO4>t(s|kM5`}hTD0$yEs8YfidqO%EzvuyT3a4JU4Gw{=7vNYp=jXZ zfgSbnDdf9)bQ_ntqlwCSA+P?|wSJVr=WF@li5XQHRRkuxgUhs~4pye~pdhA%NhA4* z52;i>eL)4bRpiV^70~M1R}(&8o!&*wnEmX#$qws_y}05hkF0m~Ti~xjqz`J-tb4Xz zJW8>rOV>|EsnmPV;+1-*<2Rig*SUXB;*qpR>@S9o8-J`vYF?;C|;27 z7xpgEo{mM!f!;mH6ZiJ`oWjq_ubLq83C~MEmTQ`eO6_HR9JkX~-S^h>G}7sQZzW&) zKl&$U0$_%ms7MG1<8aQ1{mB__E(jlxn->V=gEygnbs(sdv7O7m^&f7)Z{C2pIJiOZ zl^gU2aiTR;?B{v0@7Myf?B1yLxn+Fw!y12u6XzW9wbS98p%SuV0hQyERV2W^J_w?M zI#%K?Cca9%5=x+D@mu#0-(y_GBK%3{Niv*Zne?m*9W`BmIu3ecS$FMi&==bxVOUDz z@ptU)@3Bc{&t5=f){VZ)oFmH?Q_V5Py2ZX+44%#jc7zt~d?REUPP z1zWP0B3nG2vhyvehJ+5^=t+euT3Ptc4EAgZ`#nr*98 zSK<`3rNHn_?4uhRsbwI-gsEP8XjgYseSW!szP<_^4WaYhLu=vu=*)MXK@@cF$9H7Jkw{*8B|vvGUJ&W;6j z&h`O9wF`Q=cDR&OM#_DlCu7tyTMFLI1 z^(M!5WGM|}*E8nZFl;p$3p~mBbi^1h{~SZa_&^GyRhP__@hw?TPNSbbI^p<;Q`sZ8 z1`CL!foA~BgyWGOB2hP;Kr3tqP;Xw_r1N=Q^N+Wvr_WSWP zEz?l5a01X^LevF2SW|r-3gw3*qgk^r;>N-0l}JS-TV2mNa{8_AT%3%}%`HugY>n-V zp=P$f|8jLOHU4+{aq$@efgId?99&@d$_4(5e)D$o0PI`hASKEb{S0P!klVai7`Q{K0oyqJMq z?hxl0uD)08hR=_488@BWeA`kITXcXe{Rg{aSI3j3Un@SNU7mz!gns*`+}GY|SX=?M zVQ3zDtQOa?izzX7TT1o5A9=EEKwbY zSYNHPpi;VfXz#VA$9WBxQ6L^?tLN+l()&!$_=t2Q(@I?1zDMdE zUYmj}%_j5WWo0#-MQ|qKI`(q=v^Jeo7Fr`G03sGtd9V0WCHuc&5mGTtIKCmwD6|Z| z<;7XwHWgHuU=}g2}N(NdJCgj= zC5fcaC((l>(O=xrylO09Xs)MDN%_bb2cSSntPbTkw#v@qkJ!zJU}NuA2?`ZVeU z{E*Ihc*S5TrU2b+ZqW78_t1Mv*0!t7$$$$_FA$G(ZrL*3fEwVkg6t^6%emax0sRlkaY%j4Cs zazAwihT_}TUTh)lABS>kl#rBQYqiGSUc9l;rxHVD$z`vBoCJmK(CL21?u+e4ceWlP zpX8rSVo!R%O-gaUZ(0@bwx*A*PpeW;`t;JF;nUA)W>)MH+vv@u;xl@qvZHZasi*Vr zgk&T*=&Fhw$1)sHdz^DiNQ)aS1f_Zf1~+BKXqK>@3Aas{J;k=wEZD$LMpP;EdLKux zz7Ha7d#_e(pZGGyj=t?DdsMH)B#W(4gFou0B{5WljhBy_ZC|Q<62uYFeVr6yNz%3M zkRbFWc3NrONt=^uqKI1aT%)m8=*M772yBEJPbmLo+r5C>M&>H*%Lh&Gbigh<3N92+ zQ4kO$ezQ>;0N?=u0lXjx1g-)9r|CJnzJ^a<|IRxg?{D6L_&C77S1|BT-l-qh&jIjn zwQ{d(Wmu|_-ZL(5#g@TPQ!R>Ru3wY?$XOyxpT$MjxLsxnLohDb%a?xVG~zyTu(-&* zLIE>DCACn-2u{L9a}=b|AFqsV+QXy|IcTWq_Z_C8sd#qo>F13^zjet;6hpk=D4lJb zH*{d~tnjB*45}*GGJz9cjwy%?7bCb*R88ib{wPox57Io2i&H~69Q<4mmcp1%Id0w$ zt6h*OqYJU0J$aUnsx`~*ed9%HY7|)V%f>5R3|-0TvUXLk$ND@!+GYNH^-kg)Oczv? zIyy(B>KHbKm9VIEtmFvV!n5SuM+1?Mi!4wY`i0_`a)-yT7!&C^=kr3eZ)BP@qx9|^v3O16jyEi<(OuPESzNpA|j%HP~Z}ko9Cu(ND)Ti9%=DL4Y^zK?b_Z!Kq zsP&I*=Y6L6qVhChTX~ZuJ*0+-J*1iE>tUl_lP7z2HiB21mF7xPK`~CU<01(?waF(^ zexNy5@9NmtPXcT=3p57n<(H~gN2LF3KFAxl)S2*dzlE3k9~w^u0Du7b;C=%DuHODG zcPBG*SLc7zcwF2@K)AZ&;Q;?$!GCEy_;3U-_B*!tYzPaT0OROJ8B()kZf#ys;eI!N zLXg)JNDVc9@zpXDlVt!bMkZeIm#>gNd^&pZcp;}&e;pA#Kqd3#%9sk5Eb?{=>Tb=5 z(L_07Hy$<8qlI_APVnn(+I`o;5h8osMyd_$t}eFIOk-FjF<9R(uXbc@Ng{h3Yb_D{ zQ+1}M57!L0RHKO9%C7#ivj#7NakgaG^MYi$kVkAuj2M>*!*=7_(>sHd_9^DEONVF1 z{PIj)OiHgWYeT5R2838RcK5R_l4DodVABDfXI9LdW3)iu8?l5jZ>T6bL5aswqjPU~V1so0;x++6lK`Vd`2d|rduNf>@S zmDYaknbRdNvbxD;#u}|aKQ-{GBO1kEXI1zS&K0$!Ky9{4+Bc(Q)vNVe6t;o;wl~!t zq%Vg9>E5ozuJ}v&Lg7o@Lmmc3iN$hI3X)G&65xlVpwNI2G)Bj`zG)c~Z6B%;ge(ElFh;$L`sa zR5l%SH2292T<(F-=NUM<0v7Bh$KOBIMLqg8zj0nhRsVH9f;>r5yUGznQ>9QM%zl1Z z3tB>+Z2Ty1z)@1KIB*@t-jeCnIp4pfc4 z&%DS|YS>g=4lRzM+AMeFvo#%yy7HDL@@HXwo_Y;vNMh@Xql4Op$ye(Ha@0>7)D3ji;DZn$!SSN~tGfW5;5 zQve7s0s=U=x!^7vdJTNutP%Aa>D{Zl~776}M?c zTl-w}Uk5OrPV5raDB-Wyy(>v5VcC4OWVn>T3($CTcvXWcDES@ZWH>f@6trm^s2R_zK*Js@yMGV>Lx5)6_j>}I^Ala6(e7ViIr}^QW_;uwl z9e<(6_O_v74|MF}Yz>sAsMs!sy+G&hhwYlOf=fE*A*rQmi3H^B{Nc=^-A`=vY8HQ0 zGMNm+E@}fy|5$+$_A%;oZ`wr1Dw3}zCS8}BHX)(bT^v;^KfAzI7X4|y1o~wi?TYE; zEQ0!_;ho64!dd|jtULcRUtZGkHHn~GCbEzes8IGtRlG(1eb$QanaTWw&H5x;fG{#v z>(=CPlW`eUr#=BRs!LfJdxsq5_6l=h9Rmhy=7F>bNF%C8A`7WvxnTCIm}8DH>34E@ zTCN2j>$$q|jrx87uWT{R-fdaO z0wx!Gcua-Iop8u|1f>`@)y$UpLXVo~xpd;g6o(~N{jt==X_IZytNnL4P|5`hM)5dZkv9H}`BX-9*nF^L_GZn=hgjZx+ZCYWldWW)EhBB$1%KQdx2{b^cjmUlVmf*? zWFYOcpqlIt(LjmykJ?|qZ?~kJ&Avey1DNNnvP3Q=#{GHDLy|b4k(;iS`cc$IIRrH9 z6UUJ?zZzVC=S3c(Q0q%f`&XA=>g zKwf3>ygLl)OmX&TA;zR`w@9mba8v<7O!$f_}Ee zmPW3PUBzAYO({d!XXg5kCM-$RQKaAUVaV9lQogkimWq-XiT=zTf<;l~^x%8Ov*ygaodttmegW(=Zt62}1Qsh^H?;kWe@$#5GynL#^1H zh9&j$ijqK5tyJJZ3xa~fT#4#ZWE*|4c580@R5`Mj&nEE>ZOpJSe@WT=(7azElM zvt;Y0tc1*`k2YJGe=>Lx*6!ar+`H4gmUQ|UBJZ`E)cvBt@J^97^Vj})$EN>xM0=I= z@0HcppZ-Y&X>R!*9QerX3#Wo6oC>01+(2GQQ7(v_jG~m5G)R+2fk*qdEBsf+Vr=7L z=49+*X>Vure>b}TaMcg^Z4$%777*aCuq9dpK5hRIwq&_IlAx0?U~c0x zT#Et-*DKlL4I`?JiEhsW6I2Cky*w$xoy|ec!`PP?U53nqQ65B+A9aOe9w&&?%Y?tY z-gRW63v(|yD*QOJ)~pMQEN(QYDRHVQ*Ypp-yIRfr$YI^M8Rs7u1i8;tOUwUfZ( z(GbW_dQJvA=N5f9-|xs6~Qun0!SL;i)9A=8h&`sBUxz* zRSG3^3KXD-WrEqS;P7=iMKyVXjw&&HLKI6PwTQomdLi2T8 zk*Y}6>z?;x@cQ6AR=MQFU3}m`kC}*-WRO!U3SwfGVircfz4mR^eAYGGmA1}8KqZ{ zuBS?D>FqCm6T@8`xy)8M%Sj#d8(%4HCwDH6nbzDlz^Dm4*Q_6>dGW;UcLLM1)2Ob= zTyC`*{RB^_FH6T7#JzSeLNO+cYyBc~+KZDil-peV(?qeq$i!^(*O&TE@H8m`>19@Z z>r`<(?k2hcCtGmS0UzZsB+Hg--lhJ;xLW6hj-ao}*ABEiPiLW#)rCHJD5pAS*+NUP zPvW8GLfg_94j-pnbB%bc8a$VSpSlRr$mlmA{s^905t*$eQ?OZu$ z@}{O-@8;;!YgO}10OsE=rAl~IZm?#O;DL-WPs~NF9PaJq4)}fuGLC8H8GICnO5S1D z_FeC;+uMO($XnFU&GsNT z$IS=efcswHf2ptEnGC!unK}LYv<-j{Aip2w07E#yV91}tHq5|&9zqm)AMv?_I1a@x zJpW_MSFn{y6*se0h^Cc*c1ikf1LhH`l|1-1UR1UGRYKyH*a~9P0LI z-q{?4A}O4JeT|d){EOuW5e%Va4gbFDtHNS4@HMF9=d5=e;`w2NU7(GIa?TBA%<#Rl z%(Fee>zo(N9VeQ?jm;%LFM}}|R*(aTc59c@4k12>mQjaxJAr4_0<5!*ke|65voXik zW(pqy9Uh%kqW zWAot;zES~TlMj0b9$^E&Xb0f)J;vseXC6TlU6Ch}Mn~NCkd1$v;V6#&C4MEvoUc<^ zJUU1<3;0g01OLEYPxu|M3%QVw79-x6k&IY6E=!oVGl{fuC8-sOWh-!+U%I4zY{5%N z&Q54ZlNL&cGGOhxL}nWojp?=LQ)_EkdQ6AXQPL|l(N|V&Ro0-z(YjzCm`XLtvi)_g zSb+yLJnY&GGf-}~-;jPCK8rk#VGd@OMU+G5Mqf=%e$-*JOp#-I&G52<;1c8RewWj6 zGPrF{zsUb7Y6M|Bq4N~C#hlpe@s~xS^93Pp7&yR#GU{x6V@it7tznt zplyGs2`Zg zDXWfFyb3^N0c2u#2yCBz@?LkvlC5;=Jnx~FXN!+BbQgfOKu?>mxXJJ`SP!Cgck=iR z?zSkPdXI~=XD4ezr3+;Ae>IZebw-2|)1#=aWLj0sXGDqi=DN~+104v4VR=S_C%7m2th3&RezT5I#i#xOjlv zJpY`d{x>4S^(CChd>ruS`M7!h85W^tyS)*RR-9w>qtrd8uu;J5)f@9B&`x)CC6yz8 z%7h+zdoB`F<7I8Xm-s54_A;K9#UE)d;a9|VK#IIf+(6Bkb_8Q;ciYeb4>HTb^W#`6 z4R(9=EZ_bkhL$&EySJazy)@oSNq_1XKJr(pO0rD%el5Lvg(p=Q=GHCw*8R301+iWd@0DuZBhJfKHU z7@9#bD$dk5>|ov5&|Jb)@KkN5Z@$88HCEe}S8Y`c>!zR+ll~h<20j6Tn#VxaE)a|X z*6&T<^>XBqA&)&uTv1jx?fvZS`PRr{)d@#A+V1k_kG)ivC+HL9)4mm%Qp>2@c0uLw zkg-tPp$)38QGQ%85Df+k)5Mq@((xDeyIkWG2Wb&nSM`db8KewbX`Cc=v8QNXV}C_c z9QKUw*wRhMc25SlcGZdu5u~wNDN`z;D#)ntjN=En4S3f_Y+IM4vPv5p`&#sAYN=BT z4jjJUkOwwJgZGo&a7pr%d@akCSMMy><*Zq$1w|)Fe}u%5NiBLVXR0!EeI_YcrHS(P z@*SZf|IoG+ijC!6sUfZ~a>+8Uyi8PL?ptr%<{ZB&p05a}L2(dSA$RT9%)j_d0Y z(wkXbnfM4D{@LJZtcR)b-&j%bLjsQVpJvDp0UzdP ze*+$4`~mp?D@}X2k%!U2-x~?Qfp6r$5hOf>d}!bP4e5Xnhy2gY+=qY<9kagy?H>OH z`2XymJ;ZuwSp1EpNAefee`H@g#Ctdu{*7l&{{L%6e7J{)!uan!{Gj^p%H)U04_o=) e$Q5+|<;JcohXOwe5D>87Zz_23F#Y8Bum1yi(;}7t literal 0 HcmV?d00001 diff --git a/tests/fixtures/streetmanager_archive_section_58_2026-09_sample.zip b/tests/fixtures/streetmanager_archive_section_58_2026-09_sample.zip new file mode 100644 index 0000000000000000000000000000000000000000..42fb2981806f6669cd001c4cc3488cdd43881594 GIT binary patch literal 6834 zcmb_gcTiJ#+YKQ=0Ff4opn&ueFd-xa>7fXrhbmP8Y0^;;kg5=>6zQlmMWqV3bOZqb zK|q!!U3v$l7lAMC?z^t+xAV??`{mC3X6~IkIrooq&U2oltws!j0000mK=-++*%&_T ztPl_Y7$pV(lnGCDk!WE!3<(n!6-9}Qqu^)@j~iCE>|LBKkZ21lM;DJ9))qHBZ#dtw z5QT}NgkciGaD)X+To?uuh7(S>xQK&?i?h?iG3RtSnh*<;Rehn!{*{x)$5zv%?}^^? z>i4p~4$@c=46?i0`b&1L?=NW83lfGLo#J>z(<3J945#^z{_ZnBGspTC-P zvOj@hdoHKzdtdFt43^!f>YC$MK3ZiT*{ep5XKs;Dm46+1L)nJEHnz&SqBpCi^c}mSn5G^!RLNk})?oSF6QJ8f@IHJp`vu-TCsi%H|>UJ4?tWMgaO)&QD#W*x=PE+TO zf0;Obc;#EJmr1}2eBI0RH&lL;25&62Bj$#drfmFDi%abc%$fQ-_flrnAUR{78&eeF zg5+O%63Ye#AA6rE9aMtXwa@kBJ?hG-$a&DNB$n7?sc){tG+3WkicXrJ?S@uElZP(n zwcEiv4I`yY*nD-OqRQsXc5=cjXAb>dx$0qx+2w~H%j(H;@O9O5w!CBz4hVd z6;cc^oN}a&Hm{J^+&I6@y)0{Mj7h>t{9;|{m%E4q+qprZN~S?2`q6uoMfn#XMzH1( zYboEDcTHvYfT-EmX~jvQLI|@0)a)72FS`Mz7c{JLLZ%pe3|j}h$oHXk)su_kA$o!N z)CI%CDl%}!qE!I|0+_3N}SZZ?gp zkUwYduxeCh7|s1`J7?YOVQ^2V8>783h!!R96KtRe@!L{_s(9OcqX zo(&lFI4iJnckZ*}LO1!Yc(n+Is9QBS0OmHY{-s=^mCa9@@xW-YFZAPZ;mAmnPQZB| z-F3%`+e{SpA&R&#B^Fg&<;{wkI%#If=Eu}dQYro}8ms9E$&5z~#3Ucuw>!t?=I@%B zYZz8^EIEHupQmiQwxHlZuNFZQ=b*>%C^EN9>*dOdkB)hv;o$qbDD>f1cF{2Z44jK| zfPOwttzUJ;g`f@@>j&f-ezoa)jdLdoe8tmhO!bu?V-DZCoFtgasTSjD!orMF}Td^mjK)&~afEKhs)|&3w+_3W|Uz zZH$W!8b#kY8+(U(v18KVW(j_P=~4G~aQ1{>tjIFmQ!Tzg*`xjE4`#%Z>*tz2o!u{% znrY&#;eBd%pXK(>dDaZJvYVr0mT%0*3ycRm{c9?XFOAovF3TQ07PqQdZRtWzjKW=7 z`~vttP`}%8J=hSSRtnk{1vK$o%>>bnuFOVjOi+&;WwCDdAlH1Vj7JMylJh70!{JJr zPbA!cHr1Ib4J%w6S#H@^FGkhrsAQ*9(hzkLD~XB~frx#OD0#h?irnncfm{&=oMF#F zc42yD9D!Fffz~)8MRqap#ZWyW2sPw`SWg0Ih^()n#h9G)Yo7ck^+xM^((+i<`}sZK zIbuZ`*yB=wZ&I5gP@ju)55ldS3*qDRrlaDJ>%Q-;s~<-5iVp7G0DyD1U|i)~p22uh zO2Iw=%mev6YMYG~maBAHno538Gik-_ctBDdJXlittR7;%S!RAcOioOMT||oD^&T!a z38Q`$e)Fq3B?sz&Cr}a%)xu_;OeRZ&O^IqyEsrXI=4K)15+oG>nMreaG&%mau9)J7L2gm&ndJk;jZ_`_@Jg0dv(}sWl^U z#2*270DB8IBJAat_zU&!DAcKxOH-Li)U-s;zwv7ybG@rE6VP&(*=)M2=Gem-|7PS3 zsmyM=;Ma+SuA1wgyr}m_r8T-*drQVvs+$!2b%_dzQRmZWYEvODHAhWrqtd^O*^h2F z`QLSjnJ*gK8gVa`T|!30Qas{I>YyAPsMIq&s<7|FhqoxKgQomkg318P)$L8GC078y9!08>a%m zpPCVa5i}zP`M#2UgK5l%__QA(RzVHyj|G_;~uOvEZ~^nlUXc09p%uz{f} zNJ`+H8E!dV2DvTydo-3R402!P-m24UsK)xqQ8^YQa?lmC-0$R8M}4AynxI(85`9fZ z4n))|K>5lxT%B7R$r-FAZ)uHn$^h1B7e^S(?E>NwuaHNNW0=51Yvl@4qk-ihP|X?l%`9* zryqTPDv^O9zZ>-VJ?>*dAJ!D;a}j05m=0+(i3b5n1_ZNZh^{N9;`n_8sG*EjL<#ZK zK%!pmhAAB&v!LEXR*1n|r#pUCQ^ByS$ZfjHh22B^yanC<1e{%YuyRx3M4&=pV$~d& zap07==b~QxT{OD{(P;iA8kt-8!*6-OVUQzQjJB3UaGY&tmwJA!WL0xoL3X9Jb?467 zuv5LzZu>&DIP+ko*XH&muZnf1<^6ZtOWm_`MgB!h;?8rLtyvIeP5_O{ZRCtpIeyLK zcJ1&tw_olC-1aIlH>>kN3{qA3GTcIl@62<)uH5(YuemBbR>98dU@=s&jD1%09J%d~ zc*O2gPBby9Yq8;45FmK>@a)f;;mG5(o+V@yZ$d`-Y0QX8NFXK9$e%LGKWPSO0ViZ^ zq%aIgIFWyZW(}7S*qP81Y(p<8I#|Z(eTDG$W#S-d?(wV|Fd?R}8T-c0+Q|2u$h4`} zFn6kg_F|_xVwhfamv0Uaku5tKny9Ek!Ef(RF05>}$}-|U+2jjeq7;7GMR`;>S|A#h zS%2bmvzT?7>ggrer{M}erp(=hYSxUkLP~-ArgYFR@j^Ta5J+}MSoOMDuV>2$5zc5{ zsk{*7?RFUMA>bDLBK8d8!TVr%(r4y4bXKrxOST+MRDz|ETbeq6D@-jnOg-!}5eVMJ z&y~;-{u$2cf#Jk5FqqOZfCWIfGA({3MV_~x`7rNP;t*fGi)_EN#K>bu7V+c#sLDDw z-5QLVK<-z%4jv7`c3hNh6m82Mlj60Sx!`zuhJZ{FnVZVC!Bhf!3dJ!4mDe>t)sK6! z;m|9GSMtqV(j>lo8q|%b2rrKEC>$EPC-FpZcRF*qZqd`TNKW|Wi&xl}DkXBb=B#qR zni2L7I_W*@zJmujY-g1R{phC}8BMJ?Led>Kx1A5AC#)m&_sd?a<=Q3|UkrA0mFJjb zo;HemDQH_r$ix)(gID)+Oedh_{k$E5VGZb;9)~PT0n=w{IuBhoVemQtEy9zR6 zfNXLs)Go1iP1Y|@MHHr){+UVvPC^Vy1p8G6{bl=tiwVQgq7omM<39F?ixKUL9X(2Yo!e(O}Nm%Q9@w z-R8C^*zA?@2r@u{5}=M-?{1x+Ihae`>O#TerncCS3LFuMsQSk%w2b+(j?7 zM0VI;6)=gmeP^d*8XF~6A$ufhAjEXxeTM3CvRiLoxVLV>(rQy7wW|Ek#lZ8yvoAt^ z;g%eK<(T$rxKHFDX=dhdYiX|IwUi{ivDzy=%9g;abAhNvW{P$SYh#5ZKoLLi%4_Vx zgI98?BwSStrCcEGdsvFI*L?0&fdPCyafzJgC2gC8doO*U9t-blG2pR0f8J9h-LthayQdUH68yJa+h6E? zuzQ=jLas)XItipkq5HusRe=@!fx`ff<(zNpkxw!*P`S|}&bnG`3Y(u^`zN;dgcPkV z+PJoC)9le~1(($N@>{Kty$>rkyaMq|@VRo~LisRh^0=zz)a5%Ct96vC0@DFkELyGk zQ|;FBDtGklMwcLm{ZPr5gOuN@?5Da`eRthP3_510wk%UKsb>_Qy(@a4 zY(8$|*?W=QwlQH5R<#~_=(4zw@ug=$X8)sSba1ll&qP&=>vpIkcpjYKd4J}F@2rEt z#o$OW*iXs&y0g^{N5{WqYdFl}J4wZa5rnQl4Dm-!z?t17tY+VYY%+{rMbE77txi!& zE2w0Cl^(4F_m4Nr(7h~}ox>~!$j9sX*8a_{ z#%cTV-|h$vvf7j8>#6Ba`^QADva05oNa7lT+pL0G@4PdTVatH;u$s#Ct82e(tz`07 z(lEbC8ozT^A|;9D9XZc-1@pI?+UMFP`_n#}TsMY1z2QMQvh0&k?f52E@$u-S&J!6v zn8KwcgS-oxQoL;G-jDR5o!$xf57hCg+n+pZcKEe0XLI}eQXekF;QFddq`DsZ(zIPa zHg(S1{#BrTDOtB5peVwfT;z=O5N!@R70wv#3GM;&P#dh1(K#cLT{cijYTM5_%O6T4 zZsm!q6d|4$iTec91yaUqo9jIRKa)?>ZXn5t*_oxBo@VKGo*5mJV;QD;#C5>adSdc$ zZ=z#?6({R2e6mOOGe;Ojch{r|M&=9tnIi-vLyE!0ej1Bb|G*JKmLyb4gt>_PZ7%*k z82ysWZkk>p48{lvL-J3UUl%;qaf41&Eh6D+sxCbDXSyi}#kdd_;4WatUQ&8k;7k zsq#B*d~cVr&D)DZLnkhEnV|XdhF*OLk)x#J>rxI2*5;3l^K6t8 zeiy0Q-3Y`%t{cn%$&D6rTEZAFuk2c#JFU|QkO z5{x{ze4KI$ldIvS<_k2WNv_!J0UfI#*H-JeCyL7t;vL%x=``*&2P=_CPJ&k))WaTs zEWmJan;@VNMWBR~NyqXTQ7E#ivstx4tc>CzX{LhPbCrJ0R~Tyn)dW*=ReWRceOjjt za|Z-Pgtr(&JjIejNGWkUN4B2IGrTL=J4NB))}@FBhal*{tq4j8lo88Ft-9n*W8i?{ zDP5vd+%7^is7vr0OY)F&Qz=M2-R2~YO1bJZp-J(vPfuhVi=?FS+_JXkQ_x?o`x4x9 zJB{|$;Gyck$PMzP^ZLFrXM`2Nb#{E1!280VcC8YD2DZitCX5QC^g=A{g$6=kJ5#bD zJgz&!SU{=28fU{#5-nUx|3J>Dn^e~@0vV|ry4x|=d=1MB8E478N}YM7(s((A316)f zmMx%fs}1LiK%uBXUi9xv#Tl(!Z8yUFAJP!P3n&ge&*>zJz3}47>x~uMY%+UT%@cC9 z`7qHC{L(|9R-BOtj(pap%+1Bk;*he#Y-)x{$UD5{L8o%N;>;o(O~mz)PKW)Bhj;G*q8w(P(P1-_SwI5soO`BUPwdcq|60$vXQco`%aNO79wTYfEKv+aC_ir5^ z-O?YNcN<}gI>>{SUbuL{txD1KH9Ny>yS_9(GUk3;x@bZ} zE5FS^32MZY7t53XW;xw!&};h0z4mdj_(=M`e^m`@a(<`&*cEG8Oz68Wy&b)V$xOCn zt58OxsRu6}@{I z&kPA4?y}|(>lZQEQGq|sjgy5cS2C6btkfTJ?s+Zt;G1L254Czd2YD4O-T<}Lh=35l zKNsGFrT4dAAYr=wf7RcoF;BOezas(wfkZKc@b{x~|2G}yY0T4Q*bmGP1dsj`^IukD zr%_M$2|rLz$o>oJKW$3?Y2^OeE}Vuvy^{TaMNs|)_Fpb&r%_L*_aCVKRDYoU>nwj7 y^K=mVfyqbnAI$%USaurr^lbkDt48+&_FqqWZ8Z|o?|%&@T!n str: + return f'"{hashlib.md5(content).hexdigest()}"' + + +def _head(content: bytes, etag: str | None = None) -> httpx.Response: + return httpx.Response( + 200, + headers={"content-length": str(len(content)), "etag": etag or _etag(content)}, + ) + + +# --- ArchiveClient ---------------------------------------------------------- + + +def test_url_is_built_without_io(): + with ArchiveClient() as arc: + assert arc.url(Topic.PERMIT, 2026, 9) == f"{BASE_URL}/permit/2026/09.zip" + assert arc.url("section_58", 2022, 3) == f"{BASE_URL}/section_58/2022/03.zip" + with pytest.raises(ValueError): + arc.url(Topic.PERMIT, 2026, 13) + + +@respx.mock +def test_available_is_false_for_an_unpublished_month(): + respx.head(f"{BASE_URL}/permit/2026/09.zip").mock(return_value=httpx.Response(200)) + respx.head(f"{BASE_URL}/permit/2026/12.zip").mock(return_value=httpx.Response(404)) + with ArchiveClient() as arc: + assert arc.available(Topic.PERMIT, 2026, 9) is True + assert arc.available(Topic.PERMIT, 2026, 12) is False + + +@respx.mock +def test_files_follows_the_listing_continuation_token(): + page1 = (FIXTURES / "streetmanager_archive_listing_page1.xml").read_bytes() + page2 = (FIXTURES / "streetmanager_archive_listing_page2.xml").read_bytes() + route = respx.get(f"{BASE_URL}/").mock( + side_effect=[httpx.Response(200, content=page1), httpx.Response(200, content=page2)] + ) + with ArchiveClient() as arc: + files = arc.files(Topic.SECTION_58) + + assert route.calls[0].request.url.params["prefix"] == "section_58/" + assert "continuation-token" not in route.calls[0].request.url.params + assert route.calls[1].request.url.params["continuation-token"].startswith("1i9EeqEmy3") + assert len(files) == 55 + first, last = files[0], files[-1] + assert (first.topic, first.year, first.month) == (Topic.SECTION_58, 2022, 3) + assert (last.year, last.month, last.size) == (2026, 9, 3089473) + assert last.etag == '"5427bb520282cd020dbd5891d984e07b"' + assert last.url == S58_URL + + +@respx.mock +def test_download_streams_to_dest_and_verifies_the_etag(tmp_path): + content = SECTION_58.read_bytes() + respx.head(S58_URL).mock(return_value=_head(content)) + respx.get(S58_URL).mock(return_value=httpx.Response(200, content=content)) + with ArchiveClient() as arc: + path = arc.download(Topic.SECTION_58, 2026, 9, tmp_path / "sm" / "s58.zip") + assert path.read_bytes() == content + assert not list(tmp_path.rglob("*.part")) + + +@respx.mock +def test_download_resumes_a_part_file_with_a_range_request(tmp_path): + content = SECTION_58.read_bytes() + dest = tmp_path / "s58.zip" + (tmp_path / "s58.zip.part").write_bytes(content[:1000]) + respx.head(S58_URL).mock(return_value=_head(content)) + route = respx.get(S58_URL).mock(return_value=httpx.Response(206, content=content[1000:])) + with ArchiveClient() as arc: + arc.download(Topic.SECTION_58, 2026, 9, dest) + assert route.calls[0].request.headers["range"] == "bytes=1000-" + assert route.calls[0].request.headers["if-range"] == _etag(content) + assert dest.read_bytes() == content + + +@respx.mock +def test_download_starts_over_when_the_server_sends_the_whole_file(tmp_path): + content = SECTION_58.read_bytes() + dest = tmp_path / "s58.zip" + (tmp_path / "s58.zip.part").write_bytes(b"x" * 1000) + respx.head(S58_URL).mock(return_value=_head(content)) + respx.get(S58_URL).mock(return_value=httpx.Response(200, content=content)) + with ArchiveClient() as arc: + arc.download(Topic.SECTION_58, 2026, 9, dest) + assert dest.read_bytes() == content + + +@respx.mock +def test_download_restarts_once_when_a_resumed_part_was_from_another_copy(tmp_path): + content = SECTION_58.read_bytes() + dest = tmp_path / "s58.zip" + (tmp_path / "s58.zip.part").write_bytes(b"x" * 1000) # same server, older file + respx.head(S58_URL).mock(return_value=_head(content)) + route = respx.get(S58_URL).mock( + side_effect=[ + httpx.Response(206, content=content[1000:]), + httpx.Response(200, content=content), + ] + ) + with ArchiveClient() as arc: + arc.download(Topic.SECTION_58, 2026, 9, dest) + assert "range" not in route.calls[1].request.headers + assert dest.read_bytes() == content + + +@respx.mock +def test_download_raises_and_keeps_nothing_when_a_fresh_file_fails_its_etag(tmp_path): + content = SECTION_58.read_bytes() + respx.head(S58_URL).mock(return_value=_head(content, etag='"' + "0" * 32 + '"')) + respx.get(S58_URL).mock(return_value=httpx.Response(200, content=content)) + with ArchiveClient() as arc, pytest.raises(TransportError): + arc.download(Topic.SECTION_58, 2026, 9, tmp_path / "s58.zip") + assert list(tmp_path.iterdir()) == [] + + +@respx.mock +def test_download_of_an_unpublished_month_raises_not_found(tmp_path): + respx.head(f"{BASE_URL}/permit/2026/12.zip").mock(return_value=httpx.Response(404)) + with ArchiveClient() as arc, pytest.raises(NotFoundError): + arc.download(Topic.PERMIT, 2026, 12, tmp_path / "p.zip") + + +# --- iter_events ------------------------------------------------------------- + + +def test_iter_events_yields_every_real_event_in_a_zip(): + stats = ReadStats() + events = list(iter_events(PERMIT, stats=stats)) + assert all(isinstance(e, EventNotification) for e in events) + assert {e.event_type for e in events} == { + "PERMIT_SUBMITTED", + "PERMIT_GRANTED", + "PERMIT_REFUSED", + "PERMIT_CANCELLED", + "PERMIT_REVOKED", + "PERMIT_ALTERATION_GRANTED", + "WORK_START", + "WORK_STOP", + "WORK_START_REVERTED", + "WORK_STOP_REVERTED", + "CURRENT_TRAFFIC_MANAGEMENT_UPDATED", + } + assert stats == ReadStats(zips=1, members=11, events=11) + assert all(e.object_type == "PERMIT" and e.object_data["usrn"] for e in events) + + +def test_iter_events_reads_every_zip_under_a_folder(tmp_path): + for path in (PERMIT, ACTIVITY, SECTION_58): + (tmp_path / path.stem).mkdir() + shutil.copy(path, tmp_path / path.stem / path.name) + stats = ReadStats() + events = list(iter_events(tmp_path, stats=stats)) + assert stats.zips == 3 + assert len(events) == 11 + 6 + 8 + assert {e.object_type for e in events} == {"PERMIT", "ACTIVITY", "SECTION_58"} + + +def test_iter_events_filters_by_event_type_and_object_data(): + events = list(iter_events(SECTION_58)) + target = events[0] + ha = target.object_data["highway_authority_swa_code"] + + assert {e.event_type for e in iter_events(SECTION_58, event_types="SECTION_58_CLOSED")} == { + "SECTION_58_CLOSED" + } + by_usrn = list(iter_events(SECTION_58, usrn=int(target.object_data["usrn"]))) + assert target.event_reference in {e.event_reference for e in by_usrn} + assert all(e.object_data["usrn"] == target.object_data["usrn"] for e in by_usrn) + assert all( + e.object_data["highway_authority_swa_code"] == ha + for e in iter_events(SECTION_58, ha_org=[ha, "no-such-code"]) + ) + # Section 58 events carry no promoter, so a promoter filter matches none. + assert list(iter_events(SECTION_58, promoter_org="anything")) == [] + + +def test_iter_events_skips_and_counts_bad_zips_and_members(tmp_path): + (tmp_path / "not-a-zip.zip").write_bytes(b"this is not a zip") + with zipfile.ZipFile(tmp_path / "mixed.zip", "w") as z: + with zipfile.ZipFile(ACTIVITY) as real: + first = real.namelist()[0] + z.writestr(first, real.read(first)) + z.writestr("truncated.json", '{"event_type": ') + z.writestr("list.json", "[]") + stats = ReadStats() + events = list(iter_events(tmp_path, stats=stats)) + assert len(events) == 1 + assert stats == ReadStats(zips=1, bad_zips=1, members=3, bad_members=2, events=1)