feat(imf): multi-part release assets + one-command publish

runtime/src/interscript_ml/registry.py

@@ -29,6 +29,13 @@ class RegistryError(ValueError):
     """The index cannot be fetched/parsed, or the id is unknown."""
 
 
+@dataclass(frozen=True)
+class Part:
+    url: str
+    sha256: str
+    size: int
+
+
 @dataclass(frozen=True)
 class IndexEntry:
     id: str
@@ -38,6 +45,7 @@ class IndexEntry:
     size: int
     precision: str
     task: str
+    parts: tuple[Part, ...] = ()
 
 
 def cache_dir() -> Path:
@@ -58,14 +66,19 @@ def load_index(index_url: str | None = None) -> dict[str, IndexEntry]:
         raise RegistryError("index must be a mapping with version: 1")
     entries: dict[str, IndexEntry] = {}
     for model_id, spec in raw.get("models", {}).items():
+        parts = tuple(
+            Part(url=part["url"], sha256=part["sha256"], size=int(part.get("size", 0)))
+            for part in spec.get("parts", [])
+        )
         entries[model_id] = IndexEntry(
             id=model_id,
             filename=spec["filename"],
-            url=spec["url"],
+            url=spec.get("url", ""),
             sha256=spec["sha256"],
             size=int(spec.get("size", 0)),
             precision=spec.get("precision", "fp32"),
             task=spec.get("task", ""),
+            parts=parts,
         )
     return entries
 
@@ -78,6 +91,32 @@ def _sha256_file(path: Path) -> str:
     return digest.hexdigest()
 
 
+def _open_channel(url: str):
+    if url.startswith("file://"):
+        return open(urlparse(url).path, "rb")
+    return urllib.request.urlopen(url)
+
+
+def _download_parts(entry: IndexEntry, downloaded: Path) -> None:
+    """Stream parts into `downloaded` in index order, verifying each part's
+    sha256 as it lands. Used when the artifact exceeds GitHub's 2 GiB
+    per-asset cap; the assembled file is checked against entry.sha256 by
+    the caller, so the cache contract is identical to single-file models."""
+    with downloaded.open("ab") as out:
+        for index, part in enumerate(entry.parts):
+            digest = hashlib.sha256()
+            with _open_channel(part.url) as remote:
+                while chunk := remote.read(1024 * 1024):
+                    out.write(chunk)
+                    digest.update(chunk)
+            actual = digest.hexdigest()
+            if actual != part.sha256:
+                raise RegistryError(
+                    f"part {index} of {entry.filename} sha256 mismatch: "
+                    f"got {actual}, index says {part.sha256}"
+                )
+
+
 def resolve(model_id: str, index_url: str | None = None) -> Path:
     """Return a verified local zip path for `model_id`, downloading and
     installing into the cache when needed. Never returns an unverified
@@ -96,7 +135,9 @@ def resolve(model_id: str, index_url: str | None = None) -> Path:
     fd, tmp_name = tempfile.mkstemp(dir=target.parent, suffix=".part")
     os.close(fd)
     downloaded = Path(tmp_name)
-    if entry.url.startswith("file://"):
+    if entry.parts:
+        _download_parts(entry, downloaded)
+    elif entry.url.startswith("file://"):
         source = Path(urlparse(entry.url).path)
         if not source.is_file():
             raise RegistryError(f"channel file missing: {source}")

runtime/tests/test_registry.py

@@ -80,3 +80,91 @@ def test_model_load_by_id(tmp_path: Path) -> None:
         assert isinstance(model.translate("he", max_len=4), str)
     finally:
         os.environ.pop("INTERSCRIPT_ML_CACHE", None)
+
+
+def test_resolve_parts_assembles_and_verifies(tmp_path: Path) -> None:
+    import hashlib
+
+    zip_path = build_tiny_zip(tmp_path / "channel" / "tiny.zip")
+    blob = zip_path.read_bytes()
+    part_a, part_b = blob[: len(blob) // 2 + 3], blob[len(blob) // 2 + 3 :]
+    channel = tmp_path / "channel"
+    (channel / "tiny.zip.part-00").write_bytes(part_a)
+    (channel / "tiny.zip.part-01").write_bytes(part_b)
+    index = {
+        "version": 1,
+        "models": {
+            "tiny-1.0": {
+                "task": "translit",
+                "precision": "fp32",
+                "filename": "tiny.zip",
+                "sha256": hashlib.sha256(blob).hexdigest(),
+                "size": len(blob),
+                "parts": [
+                    {
+                        "url": f"file://{channel / 'tiny.zip.part-00'}",
+                        "sha256": hashlib.sha256(part_a).hexdigest(),
+                        "size": len(part_a),
+                    },
+                    {
+                        "url": f"file://{channel / 'tiny.zip.part-01'}",
+                        "sha256": hashlib.sha256(part_b).hexdigest(),
+                        "size": len(part_b),
+                    },
+                ],
+            },
+        },
+    }
+    index_path = tmp_path / "models.yaml"
+    index_path.write_text(yaml.safe_dump(index), encoding="utf-8")
+    cache = tmp_path / "cache"
+    os.environ["INTERSCRIPT_ML_CACHE"] = str(cache)
+    try:
+        local = resolve("tiny-1.0", index_url=str(index_path))
+        assert local == cache / "models" / "tiny-1.0" / "tiny.zip"
+        assert local.read_bytes() == blob
+        zip_path.unlink()
+        (channel / "tiny.zip.part-00").unlink()
+        assert resolve("tiny-1.0", index_url=str(index_path)) == local
+    finally:
+        os.environ.pop("INTERSCRIPT_ML_CACHE", None)
+
+
+def test_resolve_parts_rejects_corrupt_part(tmp_path: Path) -> None:
+    import hashlib
+
+    zip_path = build_tiny_zip(tmp_path / "channel" / "tiny.zip")
+    blob = zip_path.read_bytes()
+    part_a, part_b = blob[:7], blob[7:]
+    channel = tmp_path / "channel"
+    (channel / "tiny.zip.part-00").write_bytes(part_a)
+    (channel / "tiny.zip.part-01").write_bytes(part_b)
+    index = {
+        "version": 1,
+        "models": {
+            "tiny-1.0": {
+                "filename": "tiny.zip",
+                "sha256": hashlib.sha256(blob).hexdigest(),
+                "parts": [
+                    {
+                        "url": f"file://{channel / 'tiny.zip.part-00'}",
+                        "sha256": "0" * 64,
+                        "size": len(part_a),
+                    },
+                    {
+                        "url": f"file://{channel / 'tiny.zip.part-01'}",
+                        "sha256": hashlib.sha256(part_b).hexdigest(),
+                        "size": len(part_b),
+                    },
+                ],
+            },
+        },
+    }
+    index_path = tmp_path / "models.yaml"
+    index_path.write_text(yaml.safe_dump(index), encoding="utf-8")
+    os.environ["INTERSCRIPT_ML_CACHE"] = str(tmp_path / "cache")
+    try:
+        with pytest.raises(RegistryError, match="part 0"):
+            resolve("tiny-1.0", index_url=str(index_path))
+    finally:
+        os.environ.pop("INTERSCRIPT_ML_CACHE", None)