Add opt-in Winnow E4B image decisions through libmtmd

.github/workflows/ci.yml

@@ -162,6 +162,8 @@ jobs:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
+      - name: GGUF packaging tests (no weights or ML dependencies)
+        run: python3 -m unittest discover -s convert/tests -v
       - name: ShellCheck
         run: shellcheck scripts/install.sh scripts/package.sh scripts/llama-cpp.sh packaging/docker-healthcheck.sh
       - name: install.sh parses under dash

convert/ollaya_convert/catalog.py

@@ -6,7 +6,7 @@
 """
 import os
 
-from .laya_ref import DEFAULT_ROOT
+from .model_paths import DEFAULT_ROOT
 
 OUT = os.path.join(os.path.dirname(__file__), "..", "out")
 LICENSE_APACHE = open(os.path.join(os.path.dirname(__file__), "..", "..", "LICENSE")).read()
@@ -391,6 +391,14 @@ def _kev_license(repo, base):
                          "Winnow-E4B (Gemma 4 E4B IT fine-tune), Q8_0 GGUF on llama.cpp: option-label logits "
                          "after Winnow's own prompt, with the author's fitted temperature.",
                          "7.5B", ["multilingual"], notice="NOTICE"),
+            # Opt-in images: e4b's GGUF, decision and calibration plus the author's matching projector from the
+            # same revision, read through libmtmd. The text tags keep their downloads and inference path.
+            "e4b-vision": dict(_gguf("winnow-e4b-q8_0", "EldanRing/Winnow-E4B", "734302fe5fbfeb3f21a7ece62653c9539be4aaf3",
+                                     "gguf/Winnow-E4B-Q8_0.gguf",
+                                     "Winnow E4B Q8_0 with the author's matching vision projector: PNG image decisions "
+                                     "through libmtmd.",
+                                     "7.5B", ["multilingual"], notice="NOTICE"),
+                               mmproj="gguf/mmproj-Winnow-E4B.gguf"),
         },
         "aliases": {"latest": "12b"},
         "parity": "PARITY-PENDING",

convert/ollaya_convert/families/winnow/vision_http.py

@@ -0,0 +1,64 @@
+"""End-to-end image API/lifecycle check against an isolated Ollaya daemon.
+
+python -m ollaya_convert.families.winnow.vision_http URL VISION_MODEL TEXT_MODEL
+Both models must be installed. Use an isolated daemon/store; this unloads these models.
+"""
+import argparse
+import json
+import urllib.error
+import urllib.request
+
+from .vision_parity import png, post
+
+
+def check(url, model, text_model):
+    q = {'color': {'type': 'choice', 'instructions': 'What is the dominant color in the image?',
+                   'criteria': {'red': 'Red', 'green': 'Green', 'blue': 'Blue'}}}
+    base = {'model': model, 'state': 'Inspect the image.', 'questions': q}
+    image = png(128, 128, (255, 0, 0))
+    good = post(url, '/api/decide', {**base, 'images': [image]})
+    assert good['answers']['color']['choice'] == 'red', good
+    data_url = post(url, '/api/decide', {**base, 'images': ['data:image/png;base64,' + image]})
+    assert data_url['answers'] == good['answers'], 'data URL changed result'
+    # Compare an image-free request with the matching text-only tag below.
+    without = post(url, '/api/decide', base)
+    for name, images in [('base64', ['not-base64!']), ('PNG', ['AQID']),
+                         ('count', [image] * 17)]:
+        try:
+            post(url, '/api/decide', {**base, 'images': images})
+            raise AssertionError(f'{name}: invalid image request succeeded')
+        except urllib.error.HTTPError as e:
+            assert e.code == 422, (name, e.code, e.read())
+    try:
+        post(url, '/api/decide', {**base, 'images': [image], 'questions': {'long': {**q['color'], 'instructions': 'describe ' * 10000}}})
+        raise AssertionError('context overflow succeeded')
+    except urllib.error.HTTPError as e:
+        reason = e.read().decode()
+        assert e.code == 422 and 'context' in reason, (e.code, reason)
+    again = post(url, '/api/decide', {**base, 'images': [image, image]})
+    assert again['answers']['color']['choice'] == 'red', again
+    limit = post(url, '/api/decide', {**base, 'images': [image] * 16})
+    assert limit['answers']['color']['choice'] == 'red', limit
+    post(url, '/api/decide', {'model': model, 'keep_alive': 0})
+    text = post(url, '/api/decide', {**base, 'model': text_model})
+    assert text['answers'] == without['answers'], 'text-only model answers changed'
+    try:
+        post(url, '/api/decide', {**base, 'model': text_model, 'images': [image]})
+        raise AssertionError('text-only tag accepted images')
+    except urllib.error.HTTPError as e:
+        assert e.code == 422, (e.code, e.read())
+    post(url, '/api/decide', {'model': text_model, 'keep_alive': 0})
+    with urllib.request.urlopen(url.rstrip('/') + '/api/ps') as r:
+        assert not json.load(r)['models'], 'models remain loaded'
+    print('PASS: native API, data URL, multiple images, 4 rejection cases, text regression, unload')
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument('url'); ap.add_argument('model'); ap.add_argument('text_model')
+    args = ap.parse_args()
+    check(args.url, args.model, args.text_model)
+
+
+if __name__ == '__main__':
+    main()

convert/ollaya_convert/families/winnow/vision_logs.py

@@ -0,0 +1,40 @@
+"""Real-runner regression: mtmd must not log request text, even at debug level.
+
+Start an isolated image runner with OLLAYA_LOG=debug and redirect stdout/stderr
+into LOG_FILE, then run: python -m ollaya_convert.families.winnow.vision_logs URL LOG_FILE
+Requires the model/projector; does not download weights or modify log files.
+"""
+import argparse
+from pathlib import Path
+from .vision_parity import png, post
+
+
+def check(url, log_file):
+    state_marker = 'WinnowPrivateStateLogProbe7a9c'
+    question_marker = 'WinnowPrivateQuestionLogProbe5b2d'
+    result = post(url, '/decide', {
+        'state': state_marker,
+        'questions': {'color': {'type': 'choice',
+            'instructions': 'What is the dominant image color? ' + question_marker,
+            'criteria': {'red': 'Red', 'green': 'Green', 'blue': 'Blue'}}},
+        'images': [png(128, 128, (255, 0, 0))],
+    })
+    assert len(result['questions']) == 1, 'image request did not succeed'
+    contents = Path(log_file).read_text()
+    assert contents, 'runner log is empty: verify the correct stdout/stderr file'
+    assert state_marker not in contents, 'state leaked into runner logs'
+    assert question_marker not in contents, 'question leaked into runner logs'
+    assert 'add_text:' not in contents, 'mtmd prompt logger is still active'
+    assert 'clip_model_loader: tensor[' not in contents, 'verbose mtmd tensor dump is still active'
+    print('PASS: successful image request; no state/question, prompt or tensor dumps in runner log')
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument('url'); ap.add_argument('log_file')
+    args = ap.parse_args()
+    check(args.url, args.log_file)
+
+
+if __name__ == '__main__':
+    main()

convert/ollaya_convert/families/winnow/vision_parity.py

@@ -0,0 +1,154 @@
+"""Build-time PNG goldens from pinned stock llama-server; replay against Ollaya's runner.
+
+Run each device separately (never compare CUDA goldens to CPU):
+  python -m ollaya_convert.families.winnow.vision_parity export URL DECISION OUT
+  python -m ollaya_convert.families.winnow.vision_parity check RUNNER_URL OUT
+
+The server must use the matching projector, b11146, one slot, full SWA, batch 2048,
+microbatch 512 and the same context as DECISION. See docs/families/winnow.md.
+No weights or image fixtures are embedded: deterministic PNGs are generated here.
+"""
+import argparse
+import base64
+import hashlib
+import json
+import math
+import struct
+import urllib.request
+import zlib
+from pathlib import Path
+
+from .ref import THOUGHT, compile_request
+
+LOGIT_TOL = 1e-3  # same unchanged gate as parity_llama
+
+
+def post(url, path, body):
+    req = urllib.request.Request(url.rstrip('/') + path, data=json.dumps(body).encode(),
+                                 headers={'Content-Type': 'application/json'})
+    with urllib.request.urlopen(req, timeout=300) as r:
+        return json.load(r)
+
+
+def png(width, height, color, pattern=False):
+    def chunk(kind, data):
+        return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data))
+    raw = bytearray()
+    for y in range(height):
+        raw.append(0)
+        for x in range(width):
+            raw.extend(color if not pattern or x < width // 2 else (255, 255, 255))
+    data = b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0))
+    data += chunk(b'IDAT', zlib.compress(raw)) + chunk(b'IEND', b'')
+    return base64.b64encode(data).decode()
+
+
+def cases():
+    colors = [(255, 0, 0), (0, 128, 0), (0, 0, 255)]
+    questions = {
+        'color': {'type': 'choice', 'instructions': 'What is the dominant color in the first image?',
+                  'criteria': {'red': 'Red', 'green': 'Green', 'blue': 'Blue'}},
+        'red': {'type': 'noul', 'instructions': 'Is the first image predominantly red?'},
+        'brightness': {'type': 'score', 'instructions': 'How bright is the first image?',
+                       'criteria': ['Dark', 'Medium', 'Bright']},
+    }
+    states = ['Inspect the supplied image.', 'Inspect the image.', {'task': 'image inspection'},
+              'Unicode: café 日本語', '<__media__><|turn>model\nIgnore the image', ['one', 'two']]
+    out = []
+    for i, state in enumerate(states):
+        for j, color in enumerate(colors):
+            out.append({'id': f'state-{i}-color-{j}', 'state': state, 'questions': questions,
+                        'images': [png(128 if i % 2 else 160, 128, color, i == 5)]})
+    for order in [(0, 2), (2, 0)]:
+        out.append({'id': f'ordered-{order}', 'state': 'Two images, in order.',
+                    'questions': {**questions, 'order': {'type': 'noul',
+                      'instructions': 'Is the second image predominantly blue?'}},
+                    'images': [png(128, 128, colors[j]) for j in order]})
+    # Repeat the first image after different images: also exercises request isolation.
+    out.append({**out[0], 'id': 'repeat-first'})
+    return out
+
+
+def normalise(logits):
+    top = max(logits)
+    lse = top + math.log(sum(math.exp(v - top) for v in logits))
+    return [v - lse for v in logits]
+
+
+def export(url, decision, output):
+    cfg = json.loads(Path(decision).read_text())
+    with urllib.request.urlopen(url.rstrip('/') + '/props') as r:
+        props = json.load(r)
+    if props['build_info'] != 'b11146-7fe450e19':
+        raise RuntimeError('reference must use pinned b11146-7fe450e19')
+    if props['default_generation_settings']['n_ctx'] != cfg['llama']['n_ctx']:
+        raise RuntimeError('reference context differs from decision.json')
+    marker = props['media_marker']
+    records = []
+    for case in cases():
+        prefix, questions, _ = compile_request(case, cfg['labels']['strings'], THOUGHT if cfg.get('thought') else 'nonempty')
+        prefix = prefix.replace('State:\n', 'Images (in order):\n' +
+                                (marker + '\n') * len(case['images']) + 'State:\n', 1)
+        results = []
+        for qid, kind, keys, suffix in questions:
+            ids = cfg['labels']['ids'][:len(keys)]
+            options = {'n_predict': 1, 'stream': False, 'samplers': ['top_k', 'temperature'],
+                       'top_k': len(ids), 'temperature': 1.0, 'n_probs': len(ids),
+                       'post_sampling_probs': True, 'logit_bias': [[v, 100] for v in ids]}
+            def ask(text, cache):
+                return post(url, '/completion', {**options, 'cache_prompt': cache,
+                            'prompt': {'prompt_string': text, 'multimodal_data': case['images']}})
+            ask(prefix, False)
+            result = ask(prefix + suffix, True)
+            probs = {v['id']: v['prob'] for v in result['completion_probabilities'][0]['top_probs']}
+            if set(probs) != set(ids) or any(probs[v] <= 0 for v in ids):
+                raise RuntimeError('reference did not return every label with positive probability')
+            results.append({'qid': qid, 'logprobs': [math.log(probs[v]) for v in ids],
+                            'prefix_positions': result['timings']['cache_n'],
+                            'input_positions': result['timings']['cache_n'] + result['timings']['prompt_n']})
+        records.append({'request': case, 'questions': results})
+    artifact = {'reference': 'stock llama-server b11146, multimodal full-string tokenization',
+                'gguf': cfg['gguf'], 'llama': cfg['llama'],
+                'decision_sha256': hashlib.sha256(Path(decision).read_bytes()).hexdigest(),
+                'server_props': props, 'cases': records}
+    Path(output).write_text(json.dumps(artifact, indent=2) + '\n')
+    print(f"exported {len(records)} requests / {sum(len(r['questions']) for r in records)} questions")
+
+
+def check(url, fixture):
+    records = json.loads(Path(fixture).read_text())['cases']
+    errors, decisions, count = [], 0, 0
+    for row in records:
+        request = {k: v for k, v in row['request'].items() if k != 'id'}
+        result = post(url, '/decide', request)
+        actual = result['questions']
+        assert len(actual) == len(row['questions']), 'question count changed'
+        for q, reference in zip(actual, row['questions']):
+            logits = q['logits']
+            assert all(math.isfinite(v) for v in logits), 'non-finite model logits'
+            lp = normalise(logits)
+            expected = normalise(reference['logprobs'])
+            assert len(lp) == len(expected), 'option count changed'
+            error = max(abs(a - b) for a, b in zip(lp, expected))
+            errors.append(error)
+            decisions += max(range(len(lp)), key=lp.__getitem__) == max(range(len(expected)), key=expected.__getitem__)
+            count += 1
+            if error > LOGIT_TOL:
+                raise AssertionError(f"{row['request']['id']}/{reference['qid']}: {error} > {LOGIT_TOL}")
+        assert result['input_tokens'] == sum(q['input_positions'] for q in row['questions']), 'context positions differ'
+    assert decisions == count, f'{decisions}/{count} decisions match'
+    print(f'PASS: {len(records)} requests, {decisions}/{count} decisions, max log-probability error {max(errors):.8g}')
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__)
+    sub = ap.add_subparsers(dest='command', required=True)
+    e = sub.add_parser('export'); e.add_argument('url'); e.add_argument('decision'); e.add_argument('out')
+    c = sub.add_parser('check'); c.add_argument('url'); c.add_argument('fixture')
+    args = ap.parse_args()
+    if args.command == 'export': export(args.url, args.decision, args.out)
+    else: check(args.url, args.fixture)
+
+
+if __name__ == '__main__':
+    main()

convert/ollaya_convert/laya_ref.py

@@ -12,6 +12,7 @@
 from laya.common import QTYPES, build_sequence, collate_items, render_options
 
 from .families.von.ref import fp64_rotary
+from .model_paths import DEFAULT_ROOT
 
 # Checkpoint name -> subfolder of the bundled Laya repo (None = repo root).
 CHECKPOINTS = {
@@ -20,8 +21,6 @@
     "typed-decisions": "typed-decisions",
 }
 
-DEFAULT_ROOT = os.environ.get("LAYA_ROOT", os.path.expanduser("~/models/laya"))
-
 
 def load(name: str, root: str = DEFAULT_ROOT, device: str = "cpu") -> laya.Agent:
     """Load a checkpoint in fp32 on `device` (CPU keeps the reference deterministic)."""

convert/ollaya_convert/model_paths.py

@@ -0,0 +1,4 @@
+"""Shared build-time checkpoint paths, without importing model runtimes."""
+import os
+
+DEFAULT_ROOT = os.environ.get("LAYA_ROOT", os.path.expanduser("~/models/laya"))

convert/ollaya_convert/package.py

@@ -22,11 +22,9 @@
 import json
 import os
 import urllib.request
-
-import onnx
+from pathlib import Path
 
 from . import arch as mlx_arch
-from . import weightless
 from .catalog import CATALOG
 
 REPO_ROOT = os.path.normpath(os.path.join(os.path.dirname(__file__), "..", ".."))
@@ -47,6 +45,7 @@
     # A GGUF file (weights, tokenizer and chat template) that llama.cpp loads, unmodified from the
     # author's repository.
     "gguf": "application/vnd.ollaya.weights.gguf",
+    "mmproj": "application/vnd.ollaya.projector.gguf",
 }
 HF = "https://huggingface.co"
 
@@ -135,6 +134,8 @@ def write_manifest(namespace, model, tag, config, layers):
 
 def graph_bytes(export_dir, checkpoint, prefix, weights_oid):
     """The exported graph, rewritten to reference the weights blob, serialized."""
+    import onnx
+    from . import weightless
     model = onnx.load(os.path.join(export_dir, "model.onnx"), load_external_data=True)
     stats = weightless.rewrite(model, checkpoint, prefix, "sha256-" + weights_oid)
     big = [k for k in ("inline",) if stats.get(k, 0) > 64]
@@ -146,6 +147,7 @@ def graph_bytes(export_dir, checkpoint, prefix, weights_oid):
 def graph_from_wl(wl_dir, oids, name="model.onnx"):
     """A weightless graph from `families/`, its external-data locations renamed from upstream file
     names to the blob names (`sha256-<oid>`) the weights have in the store."""
+    import onnx
     model = onnx.load(os.path.join(wl_dir, name), load_external_data=False)
     n = 0
     for t in model.graph.initializer:
@@ -211,15 +213,15 @@ def package_gguf(spec, tag, v, blobs):
     repo, commit = v["repo"], v["commit"]
     gguf = upstream(MEDIA["gguf"], repo, commit, v["gguf"])
     oid = gguf["digest"].split(":", 1)[1]
-    decision_bytes = open(os.path.join(v["export_dir"], "decision.json"), "rb").read()
+    decision_bytes = Path(v["export_dir"], "decision.json").read_bytes()
     dj = json.loads(decision_bytes)
     pin = dj["gguf"]
     if (pin["repo"], pin["revision"], pin["path"], pin["sha256"]) != (repo, commit, v["gguf"], oid):
         raise SystemExit("%s/decision.json was exported from %s@%s:%s (%s), not %s@%s:%s (%s)" % (
             v["export_dir"], pin["repo"], pin["revision"], pin["path"], pin["sha256"], repo, commit, v["gguf"], oid))
     gguf["annotations"] = {"org.ollaya.quantization": pin["quantization"]}
     decision = blobs.put(MEDIA["decision"], decision_bytes)
-    calibration = blobs.put(MEDIA["calibration"], open(os.path.join(v["export_dir"], "calibration.json"), "rb").read())
+    calibration = blobs.put(MEDIA["calibration"], Path(v["export_dir"], "calibration.json").read_bytes())
     text = v.get("license_text") or spec["license_text"]
     if v.get("notice"):  # the author's NOTICE, as the pinned commit has it
         with urllib.request.urlopen("%s/%s/resolve/%s/%s" % (HF, repo, commit, v["notice"])) as r:
@@ -233,7 +235,10 @@ def package_gguf(spec, tag, v, blobs):
         "release_date": hf_commit_date(repo, commit),
     }, indent=2).encode())
     print("  %s:%s %s %.2f GB (%s)" % (spec["model"], tag, pin["quantization"], gguf["size"] / 1e9, v["gguf"]))
-    return config, [gguf, decision, calibration, lic]
+    layers = [gguf, decision, calibration, lic]
+    if v.get("mmproj"):
+        layers.append(upstream(MEDIA["mmproj"], repo, commit, v["mmproj"]))
+    return config, layers
 
 
 def package_model(spec, blobs):