From 407e85922a2dd4fbb29cc067d18de81159b16de5 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Tue, 18 Aug 2026 16:22:51 +0800 Subject: [PATCH] =?UTF-8?q?feat(imf):=20multi-part=20release=20assets=20?= =?UTF-8?q?=E2=80=94=20GitHub=202GiB=20cap=20+=20one-command=20publish?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit heb-diac-1.0 fp32 (2.59 GiB) exceeds GitHub's hard 2GiB per-asset cap. models.yaml entries may now carry parts: [{url, sha256, size}]; the registry streams parts in order, verifies each sha256 as it lands, and checks the assembled file against the whole-file sha256 — the cache contract is identical to single-file models. - scripts/split_release.py: split + per-part sha256 + models.yaml block - scripts/publish_model.py: validate (strict) -> split -> GH Release -> models.yaml entry -> release branch -> PR; idempotent re-runs - runtime registry: parts resolution + tests (assembly, corrupt-part rejection, verified cache hit) --- runtime/src/interscript_ml/registry.py | 45 ++++- runtime/tests/test_registry.py | 88 +++++++++ scripts/publish_model.py | 236 +++++++++++++++++++++++++ scripts/split_release.py | 83 +++++++++ 4 files changed, 450 insertions(+), 2 deletions(-) create mode 100644 scripts/publish_model.py create mode 100644 scripts/split_release.py diff --git a/runtime/src/interscript_ml/registry.py b/runtime/src/interscript_ml/registry.py index c53fcb1..1833281 100644 --- a/runtime/src/interscript_ml/registry.py +++ b/runtime/src/interscript_ml/registry.py @@ -29,6 +29,13 @@ class RegistryError(ValueError): """The index cannot be fetched/parsed, or the id is unknown.""" +@dataclass(frozen=True) +class Part: + url: str + sha256: str + size: int + + @dataclass(frozen=True) class IndexEntry: id: str @@ -38,6 +45,7 @@ class IndexEntry: size: int precision: str task: str + parts: tuple[Part, ...] = () def cache_dir() -> Path: @@ -58,14 +66,19 @@ def load_index(index_url: str | None = None) -> dict[str, IndexEntry]: raise RegistryError("index must be a mapping with version: 1") entries: dict[str, IndexEntry] = {} for model_id, spec in raw.get("models", {}).items(): + parts = tuple( + Part(url=part["url"], sha256=part["sha256"], size=int(part.get("size", 0))) + for part in spec.get("parts", []) + ) entries[model_id] = IndexEntry( id=model_id, filename=spec["filename"], - url=spec["url"], + url=spec.get("url", ""), sha256=spec["sha256"], size=int(spec.get("size", 0)), precision=spec.get("precision", "fp32"), task=spec.get("task", ""), + parts=parts, ) return entries @@ -78,6 +91,32 @@ def _sha256_file(path: Path) -> str: return digest.hexdigest() +def _open_channel(url: str): + if url.startswith("file://"): + return open(urlparse(url).path, "rb") + return urllib.request.urlopen(url) + + +def _download_parts(entry: IndexEntry, downloaded: Path) -> None: + """Stream parts into `downloaded` in index order, verifying each part's + sha256 as it lands. Used when the artifact exceeds GitHub's 2 GiB + per-asset cap; the assembled file is checked against entry.sha256 by + the caller, so the cache contract is identical to single-file models.""" + with downloaded.open("ab") as out: + for index, part in enumerate(entry.parts): + digest = hashlib.sha256() + with _open_channel(part.url) as remote: + while chunk := remote.read(1024 * 1024): + out.write(chunk) + digest.update(chunk) + actual = digest.hexdigest() + if actual != part.sha256: + raise RegistryError( + f"part {index} of {entry.filename} sha256 mismatch: " + f"got {actual}, index says {part.sha256}" + ) + + def resolve(model_id: str, index_url: str | None = None) -> Path: """Return a verified local zip path for `model_id`, downloading and installing into the cache when needed. Never returns an unverified @@ -96,7 +135,9 @@ def resolve(model_id: str, index_url: str | None = None) -> Path: fd, tmp_name = tempfile.mkstemp(dir=target.parent, suffix=".part") os.close(fd) downloaded = Path(tmp_name) - if entry.url.startswith("file://"): + if entry.parts: + _download_parts(entry, downloaded) + elif entry.url.startswith("file://"): source = Path(urlparse(entry.url).path) if not source.is_file(): raise RegistryError(f"channel file missing: {source}") diff --git a/runtime/tests/test_registry.py b/runtime/tests/test_registry.py index 25c8b1a..2e9b996 100644 --- a/runtime/tests/test_registry.py +++ b/runtime/tests/test_registry.py @@ -80,3 +80,91 @@ def test_model_load_by_id(tmp_path: Path) -> None: assert isinstance(model.translate("he", max_len=4), str) finally: os.environ.pop("INTERSCRIPT_ML_CACHE", None) + + +def test_resolve_parts_assembles_and_verifies(tmp_path: Path) -> None: + import hashlib + + zip_path = build_tiny_zip(tmp_path / "channel" / "tiny.zip") + blob = zip_path.read_bytes() + part_a, part_b = blob[: len(blob) // 2 + 3], blob[len(blob) // 2 + 3 :] + channel = tmp_path / "channel" + (channel / "tiny.zip.part-00").write_bytes(part_a) + (channel / "tiny.zip.part-01").write_bytes(part_b) + index = { + "version": 1, + "models": { + "tiny-1.0": { + "task": "translit", + "precision": "fp32", + "filename": "tiny.zip", + "sha256": hashlib.sha256(blob).hexdigest(), + "size": len(blob), + "parts": [ + { + "url": f"file://{channel / 'tiny.zip.part-00'}", + "sha256": hashlib.sha256(part_a).hexdigest(), + "size": len(part_a), + }, + { + "url": f"file://{channel / 'tiny.zip.part-01'}", + "sha256": hashlib.sha256(part_b).hexdigest(), + "size": len(part_b), + }, + ], + }, + }, + } + index_path = tmp_path / "models.yaml" + index_path.write_text(yaml.safe_dump(index), encoding="utf-8") + cache = tmp_path / "cache" + os.environ["INTERSCRIPT_ML_CACHE"] = str(cache) + try: + local = resolve("tiny-1.0", index_url=str(index_path)) + assert local == cache / "models" / "tiny-1.0" / "tiny.zip" + assert local.read_bytes() == blob + zip_path.unlink() + (channel / "tiny.zip.part-00").unlink() + assert resolve("tiny-1.0", index_url=str(index_path)) == local + finally: + os.environ.pop("INTERSCRIPT_ML_CACHE", None) + + +def test_resolve_parts_rejects_corrupt_part(tmp_path: Path) -> None: + import hashlib + + zip_path = build_tiny_zip(tmp_path / "channel" / "tiny.zip") + blob = zip_path.read_bytes() + part_a, part_b = blob[:7], blob[7:] + channel = tmp_path / "channel" + (channel / "tiny.zip.part-00").write_bytes(part_a) + (channel / "tiny.zip.part-01").write_bytes(part_b) + index = { + "version": 1, + "models": { + "tiny-1.0": { + "filename": "tiny.zip", + "sha256": hashlib.sha256(blob).hexdigest(), + "parts": [ + { + "url": f"file://{channel / 'tiny.zip.part-00'}", + "sha256": "0" * 64, + "size": len(part_a), + }, + { + "url": f"file://{channel / 'tiny.zip.part-01'}", + "sha256": hashlib.sha256(part_b).hexdigest(), + "size": len(part_b), + }, + ], + }, + }, + } + index_path = tmp_path / "models.yaml" + index_path.write_text(yaml.safe_dump(index), encoding="utf-8") + os.environ["INTERSCRIPT_ML_CACHE"] = str(tmp_path / "cache") + try: + with pytest.raises(RegistryError, match="part 0"): + resolve("tiny-1.0", index_url=str(index_path)) + finally: + os.environ.pop("INTERSCRIPT_ML_CACHE", None) diff --git a/scripts/publish_model.py b/scripts/publish_model.py new file mode 100644 index 0000000..589133c --- /dev/null +++ b/scripts/publish_model.py @@ -0,0 +1,236 @@ +#!/usr/bin/env python3 +"""Publish an IMF zip: validate -> (split if >2GiB) -> GH Release -> +models.yaml entry -> PR. One command, the whole tail of the pipeline. + + python scripts/publish_model.py heb-diac-1.0 --zip /tmp/heb-diac-1.0-fp32.zip + +Idempotent: an existing release gets assets re-uploaded (clobber), an +existing models.yaml entry block is replaced in place, an existing +release branch/PR is reused. Never touches main directly. + +The metadata inside the zip is the source of truth for metrics/parity; +the strict validator gate must pass before anything is uploaded. +""" + +from __future__ import annotations + +import argparse +import hashlib +import os +import re +import subprocess +import sys +import tempfile +import zipfile +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT / "src")) +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +from split_release import split as split_zip # noqa: E402 +from imf.validator import validate_zip # noqa: E402 + +# GitHub hard-caps release assets at 2,147,483,648 bytes; split well below. +SPLIT_THRESHOLD = 2_000_000_000 +DEFAULT_REPO = "interscript/ml-models" + + +def run(cmd: list[str], **kwargs) -> subprocess.CompletedProcess: + return subprocess.run(cmd, check=True, **kwargs) + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as fh: + while chunk := fh.read(1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def load_metadata(zip_path: Path) -> dict: + with zipfile.ZipFile(zip_path) as zf: + return yaml.safe_load(zf.read("metadata.yaml")) + + +def entry_block(model_id: str, meta: dict, filename: str, sha256: str, size: int, + assets: list[Path], repo: str, tag: str) -> str: + base = f"https://github.com/{repo}/releases/download/{tag}" + lines = [ + f" {model_id}:", + f" task: {meta['task']}", + f" scripts: [{meta['source_script']}, {meta['target']}]", + f" precision: {meta['precision']}", + f" filename: {filename}", + ] + if len(assets) == 1: + lines.append(f" url: {base}/{assets[0].name}") + else: + lines.append(" parts:") + for asset in assets: + lines += [ + f" - url: {base}/{asset.name}", + f" sha256: {sha256_file(asset)}", + f" size: {asset.stat().st_size}", + ] + lines += [ + f" sha256: {sha256}", + f" size: {size}", + " metrics:", + ] + for metric in meta["metrics"]: + lines.append( + f" - {{name: {metric['name']}, value: {metric['value']}, " + f"source: {metric['source']}}}" + ) + parity = meta["parity"] + lines += [ + f" parity: {{samples: {parity['samples']}, cer_delta: {parity['cer_delta']}}}", + f" license: {meta['license']}", + ] + return "\n".join(lines) + "\n" + + +def upsert_models_yaml(model_id: str, block: str) -> None: + path = REPO_ROOT / "models.yaml" + text = path.read_text(encoding="utf-8") + pattern = re.compile( + rf"^ {re.escape(model_id)}:\n(?:(?!^ \S).*\n)*", re.MULTILINE + ) + if pattern.search(text): + path.write_text(pattern.sub(block, text), encoding="utf-8") + else: + with path.open("a", encoding="utf-8") as fh: + fh.write(block) + + +def release_notes(model_id: str, meta: dict, filename: str, size: int, + sha256: str, assets: list[Path]) -> str: + lines = [ + f"# {model_id}", + "", + f"IMF v1 (`{meta['precision']}`, decoder `{meta['decoder']}`, opset " + f"{meta['opset']}). Trained from {meta['trained_from']}.", + "", + "| field | value |", + "|---|---|", + f"| task | {meta['task']} ({meta['source_script']} → {meta['target']}) |", + f"| artifact | {filename} ({size / 1024**3:.2f} GiB)", + ] + if len(assets) > 1: + lines.append(f"| parts | {' + '.join(a.name for a in assets)} (GitHub 2GiB cap) |") + for metric in meta["metrics"]: + lines.append(f"| {metric['name']} | {metric['value']} — {metric['protocol']} |") + parity = meta["parity"] + lines += [ + f"| parity | cer_delta {parity['cer_delta']}pp on {parity['samples']} samples |", + f"| sha256 | `{sha256}` |", + f"| license | {meta['license']} |", + "", + "Runtimes reassemble split parts transparently and verify every sha256:", + "", + "```python", + "from interscript_ml import Model", + f'model = Model.load("{model_id}")', + "```", + ] + return "\n".join(lines) + "\n" + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("model_id") + parser.add_argument("--zip", type=Path, required=True) + parser.add_argument("--repo", default=DEFAULT_REPO) + args = parser.parse_args() + + # git and gh must operate on this repo regardless of the caller's cwd + os.chdir(REPO_ROOT) + + if args.zip.resolve().parent == REPO_ROOT: + raise SystemExit("refusing to publish from the repo root; keep zips outside the tree") + + result = validate_zip(args.zip, strict=True) + if not result.ok: + result.errors and print("\n".join(result.errors), file=sys.stderr) + raise SystemExit("strict validation failed; nothing published") + + meta = load_metadata(args.zip) + if meta["id"] != args.model_id: + raise SystemExit(f"metadata id {meta['id']!r} != requested {args.model_id!r}") + + whole_sha = sha256_file(args.zip) + size = args.zip.stat().st_size + + assets = [args.zip] + if size > SPLIT_THRESHOLD: + print(f"{size:,} bytes > {SPLIT_THRESHOLD:,}; splitting for the GitHub asset cap") + parts = split_zip(args.zip, 1_500_000_000) + assets = [part for part, _, _ in parts] + + tag = args.model_id + with tempfile.NamedTemporaryFile("w", suffix=".md", delete=False) as fh: + fh.write(release_notes(args.model_id, meta, args.zip.name, size, whole_sha, assets)) + notes_path = fh.name + + existing = subprocess.run( + ["gh", "release", "view", tag, "--json", "name"], + capture_output=True, text=True, + ) + if existing.returncode == 0: + print(f"release {tag} exists; re-uploading assets (clobber)") + run(["gh", "release", "upload", tag, *[str(a) for a in assets], "--clobber"]) + run(["gh", "release", "edit", tag, "--notes-file", notes_path]) + else: + run(["gh", "release", "create", tag, *[str(a) for a in assets], + "--title", tag, "--notes-file", notes_path]) + + upsert_models_yaml(args.model_id, + entry_block(args.model_id, meta, args.zip.name, whole_sha, size, + assets, args.repo, tag)) + + model_dir = REPO_ROOT / "models" / args.model_id + model_dir.mkdir(parents=True, exist_ok=True) + (model_dir / f"{args.model_id}.metadata.yaml").write_text( + yaml.safe_dump(meta, sort_keys=False, allow_unicode=True), encoding="utf-8") + + branch = f"release/{args.model_id}" + current = run(["git", "rev-parse", "--abbrev-ref", "HEAD"], + capture_output=True, text=True).stdout.strip() + if current != branch: + branches = run(["git", "branch", "--list", branch], + capture_output=True, text=True).stdout + run(["git", "checkout", "-B", branch, "origin/main"] if not branches else + ["git", "checkout", branch]) + run(["git", "add", "models.yaml", + str(model_dir / f"{args.model_id}.metadata.yaml")]) + staged = run(["git", "diff", "--cached", "--name-only"], + capture_output=True, text=True).stdout.split() + if not staged: + print("nothing new to commit (idempotent re-run)") + return + run(["git", "commit", "-m", f"release: {args.model_id} " + f"({meta['precision']}, parity cer_delta {meta['parity']['cer_delta']}pp " + f"on {meta['parity']['samples']} samples)"]) + run(["git", "push", "-u", "origin", branch]) + prs = run(["gh", "pr", "list", "--head", branch, "--json", "number"], + capture_output=True, text=True).stdout + if "number" not in prs or yaml.safe_load(prs) == []: + with tempfile.NamedTemporaryFile("w", suffix=".md", delete=False) as fh: + fh.write(f"## Summary\n- publish {args.model_id} ({meta['precision']}): " + f"GH Release `{tag}` + models.yaml entry" + f"{' (split parts, GitHub 2GiB cap)' if len(assets) > 1 else ''}\n" + f"- parity cer_delta {meta['parity']['cer_delta']}pp on " + f"{meta['parity']['samples']} samples; strict validator gate passed\n\n" + f"## Test plan\n- [ ] CI green\n- [ ] runtime fetch " + f"`Model.load(\"{args.model_id}\")` resolves and verifies\n") + body_path = fh.name + run(["gh", "pr", "create", "--title", f"release: {args.model_id}", + "--body-file", body_path]) + print(f"published {args.model_id}: release {tag}, PR on {branch}") + + +if __name__ == "__main__": + main() diff --git a/scripts/split_release.py b/scripts/split_release.py new file mode 100644 index 0000000..4c289f0 --- /dev/null +++ b/scripts/split_release.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Split a release asset to fit GitHub's 2 GiB per-asset cap. + +Writes ``.part-00``, ``.part-01``, ... alongside the source, +prints per-part sha256s, and emits a ``parts:`` block for models.yaml. +Runtimes reassemble by plain byte concatenation; the whole-file sha256 +remains the index contract, each part carries its own sha256 so a +corrupt part is identified, not just "the download failed". + + python scripts/split_release.py models/heb-diac/heb-diac-1.0-fp32.zip \ + --url-base https://github.com/interscript/ml-models/releases/download/heb-diac-1.0 +""" + +from __future__ import annotations + +import argparse +import hashlib +from pathlib import Path + +# GitHub hard-caps release assets at 2,147,483,648 bytes; stay well clear. +DEFAULT_PART_SIZE = 1_500_000_000 +CHUNK = 1024 * 1024 + + +def split(src: Path, part_size: int) -> list[tuple[Path, str, int]]: + parts: list[tuple[Path, str, int]] = [] + with src.open("rb") as fh: + index = 0 + while True: + remaining = part_size + digest = hashlib.sha256() + part_path = src.parent / f"{src.name}.part-{index:02d}" + written = 0 + with part_path.open("wb") as out: + while remaining > 0: + chunk = fh.read(min(CHUNK, remaining)) + if not chunk: + break + out.write(chunk) + digest.update(chunk) + remaining -= len(chunk) + written += len(chunk) + if written == 0: + part_path.unlink() + break + parts.append((part_path, digest.hexdigest(), written)) + print(f"{part_path.name} {written:>13,} bytes {digest.hexdigest()}") + index += 1 + if written < part_size: + break + return parts + + +def yaml_block(parts: list[tuple[Path, str, int]], url_base: str) -> str: + lines = [" parts:"] + for part_path, sha256, size in parts: + url = f"{url_base.rstrip('/')}/{part_path.name}" if url_base else part_path.name + lines.append(f" - url: {url}") + lines.append(f" sha256: {sha256}") + lines.append(f" size: {size}") + return "\n".join(lines) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("zip", type=Path) + parser.add_argument("--part-size", type=int, default=DEFAULT_PART_SIZE) + parser.add_argument("--url-base", default="") + args = parser.parse_args() + + src = args.zip.resolve() + parts = split(src, args.part_size) + total = hashlib.sha256() + with src.open("rb") as fh: + while chunk := fh.read(CHUNK): + total.update(chunk) + print(f"\nwhole-file sha256: {total.hexdigest()} ({src.stat().st_size:,} bytes)") + print("\nmodels.yaml entry:") + print(yaml_block(parts, args.url_base)) + + +if __name__ == "__main__": + main()