From 8524ba236f06f818d94e22bfd91312d4cd3e3e31 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Tue, 18 Aug 2026 17:04:42 +0800 Subject: [PATCH 1/4] fix(distill): lint on main + Thai teacher recovery + parts-aware publish MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ruff: unused val_path/epoch/math in modal_distill/modal_teacher_thai - modal_teacher_thai.py: recover the Thai umt5 teacher under transformers 5.14.1 — every secryst-saved umt5 artifact is unusable (5.15 save dropped the untied lm_head; verified by probes 2026-08-18). Same recipe as train_thai_combined.py: B-K base + 60K Kaikki+epitran. - decode_joined everywhere: 5.x batch_decode inserts spurious spaces between sentencepiece pieces — this mangled the first student's teacher labels; tha-g2p-small moves to run-002 (retrain pending) - publish_model.py: worktree-based branch handling (dirty trees no longer block publication), skip re-upload when assets already match --- .../tha-g2p-small/tha-g2p-small-1.0.README.md | 17 ++ .../tha-g2p-small-1.0.metadata.yaml | 30 +++ scripts/publish_model.py | 115 ++++++---- src/gpu/modal_distill.py | 118 +++++++++- src/gpu/modal_export.py | 9 + src/gpu/modal_teacher_thai.py | 215 ++++++++++++++++++ 6 files changed, 450 insertions(+), 54 deletions(-) create mode 100644 models/tha-g2p-small/tha-g2p-small-1.0.README.md create mode 100644 models/tha-g2p-small/tha-g2p-small-1.0.metadata.yaml create mode 100644 src/gpu/modal_teacher_thai.py diff --git a/models/tha-g2p-small/tha-g2p-small-1.0.README.md b/models/tha-g2p-small/tha-g2p-small-1.0.README.md new file mode 100644 index 0000000..469d477 --- /dev/null +++ b/models/tha-g2p-small/tha-g2p-small-1.0.README.md @@ -0,0 +1,17 @@ +# tha-g2p-small-1.0 + +Thai grapheme-to-phoneme (IPA). Client-tier ByT5-small (300M) student +distilled from the secryst umt5 Thai teacher (2.32% PER, public baseline +6.37%) via sequence-level KD: the teacher generated 23,295 labels with +its own sentencepiece tokenizer and the student trained CE on them with +the canonical byte table. + +First model of the distillation campaign +(docs/DISTILL-SOURCE-PROMPT.md); identical IMF v1 contract to the +server-tier models — dynamic fetch, sha256-verified, KV decode. + +```python +from interscript_ml import Model +model = Model.load("tha-g2p-small-1.0") +model.translate("สวัสดี") +``` diff --git a/models/tha-g2p-small/tha-g2p-small-1.0.metadata.yaml b/models/tha-g2p-small/tha-g2p-small-1.0.metadata.yaml new file mode 100644 index 0000000..de51569 --- /dev/null +++ b/models/tha-g2p-small/tha-g2p-small-1.0.metadata.yaml @@ -0,0 +1,30 @@ +format: imf-v1 +id: tha-g2p-small-1.0 +task: g2p +source_script: Thai +target: IPA +tokenizer: bytes +opset: 14 +decoder: kv +precision: fp32 +license: BSD-3-Clause +trained_from: >- + distilled from the recovered Thai umt5 teacher (transformers 5.14.1 + re-finetune of B-K/umt5-thai-g2p-v2-0.5k on the 60K Kaikki+epitran + corpus; the secryst-saved umt5 artifacts are unusable — 5.15 dropped + the untied lm_head) via sequence-level KD; ByT5-small init + google/byt5-small; checkpoint + secryst-checkpoints:/secryst_thai_g2p_distill_small/run-002/best +metrics: + - name: per_teacher + value: 2.32 + protocol: >- + greedy decode; 1,219 Kaikki Thai test sentences; umt5 teacher, + sentencepiece tokenizer; secryst RESULTS.md protocol + source: secryst/docs/RESULTS.md#thai-g2p + - name: per_student + value: PLACEHOLDER_STUDENT_PER + protocol: >- + greedy decode; same 1,219 test sentences, same harness as the + teacher (interscript/ml-models src/gpu/modal_distill.py::evaluate_per) + source: interscript/ml-models release tha-g2p-small-1.0 diff --git a/scripts/publish_model.py b/scripts/publish_model.py index 589133c..1c8b826 100644 --- a/scripts/publish_model.py +++ b/scripts/publish_model.py @@ -93,16 +93,15 @@ def entry_block(model_id: str, meta: dict, filename: str, sha256: str, size: int return "\n".join(lines) + "\n" -def upsert_models_yaml(model_id: str, block: str) -> None: - path = REPO_ROOT / "models.yaml" - text = path.read_text(encoding="utf-8") +def upsert_models_yaml(models_yaml: Path, model_id: str, block: str) -> None: + text = models_yaml.read_text(encoding="utf-8") pattern = re.compile( rf"^ {re.escape(model_id)}:\n(?:(?!^ \S).*\n)*", re.MULTILINE ) if pattern.search(text): - path.write_text(pattern.sub(block, text), encoding="utf-8") + models_yaml.write_text(pattern.sub(block, text), encoding="utf-8") else: - with path.open("a", encoding="utf-8") as fh: + with models_yaml.open("a", encoding="utf-8") as fh: fh.write(block) @@ -180,56 +179,74 @@ def main() -> None: capture_output=True, text=True, ) if existing.returncode == 0: - print(f"release {tag} exists; re-uploading assets (clobber)") - run(["gh", "release", "upload", tag, *[str(a) for a in assets], "--clobber"]) + listed = run(["gh", "release", "view", tag, "--json", "assets", + "--jq", "[.assets[] | {name, size}]"], + capture_output=True, text=True).stdout + have = { + asset["name"]: int(asset["size"]) + for asset in yaml.safe_load(listed) + } + want = {a.name: a.stat().st_size for a in assets} + if have == want: + print(f"release {tag} already carries all assets; skipping upload") + else: + print(f"release {tag} exists; re-uploading assets (clobber)") + run(["gh", "release", "upload", tag, *[str(a) for a in assets], "--clobber"]) run(["gh", "release", "edit", tag, "--notes-file", notes_path]) else: run(["gh", "release", "create", tag, *[str(a) for a in assets], "--title", tag, "--notes-file", notes_path]) - upsert_models_yaml(args.model_id, - entry_block(args.model_id, meta, args.zip.name, whole_sha, size, - assets, args.repo, tag)) - - model_dir = REPO_ROOT / "models" / args.model_id - model_dir.mkdir(parents=True, exist_ok=True) - (model_dir / f"{args.model_id}.metadata.yaml").write_text( - yaml.safe_dump(meta, sort_keys=False, allow_unicode=True), encoding="utf-8") - + # Branch work happens in a dedicated worktree so the caller's tree is + # never checked out (dirty files must not block publication, and + # publication must not clobber in-progress edits). branch = f"release/{args.model_id}" - current = run(["git", "rev-parse", "--abbrev-ref", "HEAD"], - capture_output=True, text=True).stdout.strip() - if current != branch: - branches = run(["git", "branch", "--list", branch], - capture_output=True, text=True).stdout - run(["git", "checkout", "-B", branch, "origin/main"] if not branches else - ["git", "checkout", branch]) - run(["git", "add", "models.yaml", - str(model_dir / f"{args.model_id}.metadata.yaml")]) - staged = run(["git", "diff", "--cached", "--name-only"], - capture_output=True, text=True).stdout.split() - if not staged: - print("nothing new to commit (idempotent re-run)") - return - run(["git", "commit", "-m", f"release: {args.model_id} " - f"({meta['precision']}, parity cer_delta {meta['parity']['cer_delta']}pp " - f"on {meta['parity']['samples']} samples)"]) - run(["git", "push", "-u", "origin", branch]) - prs = run(["gh", "pr", "list", "--head", branch, "--json", "number"], - capture_output=True, text=True).stdout - if "number" not in prs or yaml.safe_load(prs) == []: - with tempfile.NamedTemporaryFile("w", suffix=".md", delete=False) as fh: - fh.write(f"## Summary\n- publish {args.model_id} ({meta['precision']}): " - f"GH Release `{tag}` + models.yaml entry" - f"{' (split parts, GitHub 2GiB cap)' if len(assets) > 1 else ''}\n" - f"- parity cer_delta {meta['parity']['cer_delta']}pp on " - f"{meta['parity']['samples']} samples; strict validator gate passed\n\n" - f"## Test plan\n- [ ] CI green\n- [ ] runtime fetch " - f"`Model.load(\"{args.model_id}\")` resolves and verifies\n") - body_path = fh.name - run(["gh", "pr", "create", "--title", f"release: {args.model_id}", - "--body-file", body_path]) - print(f"published {args.model_id}: release {tag}, PR on {branch}") + worktree = REPO_ROOT.parent / f".wt-publish-{args.model_id}" + branches = run(["git", "branch", "--list", branch], + capture_output=True, text=True).stdout + base = branch if branches else "origin/main" + run(["git", "worktree", "add", str(worktree), "-B", branch, base]) + + try: + wt_models = worktree / "models.yaml" + wt_repo = str(worktree) + upsert_models_yaml(wt_models, args.model_id, + entry_block(args.model_id, meta, args.zip.name, whole_sha, + size, assets, args.repo, tag)) + model_dir = worktree / "models" / args.model_id + model_dir.mkdir(parents=True, exist_ok=True) + (model_dir / f"{args.model_id}.metadata.yaml").write_text( + yaml.safe_dump(meta, sort_keys=False, allow_unicode=True), encoding="utf-8") + + def git_wt(*cmd: str): + return run(["git", "-C", wt_repo, *cmd], capture_output=True, text=True) + + git_wt("add", "models.yaml", f"models/{args.model_id}/{args.model_id}.metadata.yaml") + staged = git_wt("diff", "--cached", "--name-only").stdout.split() + if staged: + git_wt("commit", "-m", f"release: {args.model_id} " + f"({meta['precision']}, parity cer_delta {meta['parity']['cer_delta']}pp " + f"on {meta['parity']['samples']} samples)") + git_wt("push", "-u", "origin", branch) + else: + print("nothing new to commit (idempotent re-run)") + prs = run(["gh", "pr", "list", "--head", branch, "--json", "number"], + capture_output=True, text=True).stdout + if not yaml.safe_load(prs): + with tempfile.NamedTemporaryFile("w", suffix=".md", delete=False) as fh: + fh.write(f"## Summary\n- publish {args.model_id} ({meta['precision']}): " + f"GH Release `{tag}` + models.yaml entry" + f"{' (split parts, GitHub 2GiB cap)' if len(assets) > 1 else ''}\n" + f"- parity cer_delta {meta['parity']['cer_delta']}pp on " + f"{meta['parity']['samples']} samples; strict validator gate passed\n\n" + f"## Test plan\n- [ ] CI green\n- [ ] runtime fetch " + f"`Model.load(\"{args.model_id}\")` resolves and verifies\n") + body_path = fh.name + run(["gh", "pr", "create", "--title", f"release: {args.model_id}", + "--body-file", body_path]) + finally: + run(["git", "worktree", "remove", "--force", str(worktree)]) + print(f"published {args.model_id}: release {tag}, branch {branch}") if __name__ == "__main__": diff --git a/src/gpu/modal_distill.py b/src/gpu/modal_distill.py index 9dbe2ed..c2447ee 100644 --- a/src/gpu/modal_distill.py +++ b/src/gpu/modal_distill.py @@ -43,13 +43,17 @@ SPECS: dict[str, dict[str, str]] = { "tha-g2p-small": { - "teacher": "secryst_thai_ipa_thai_combined_mixed/run-001/best", + # secryst's saved umt5 artifacts are unusable (5.15 dropped the + # untied lm_head) — teacher is the 5.14.1 recovery finetune + # (src/gpu/modal_teacher_thai.py, same recipe + data) + "teacher": "secryst_thai_ipa_teacher_recovery/run-001/best", "teacher_volume": "secryst", "student_init": "google/byt5-small", "train": "thai-ipa-expanded/train.jsonl", "val": "thai-ipa-expanded/val.jsonl", "test": "thai-ipa-expanded/test.jsonl", - "out": "secryst_thai_g2p_distill_small/run-001", + "eval_test": "thai-ipa/test.jsonl", + "out": "secryst_thai_g2p_distill_small/run-002", "mode": "sequence", # cross-tokenizer: teacher generates, student trains CE "note": "umt5 (sentencepiece) teacher -> ByT5-small byte student; +5pp PER gate", }, @@ -75,6 +79,14 @@ app = modal.App("interscript-ml-distill", image=IMAGE) +def decode_joined(tok, ids) -> str: + """Correct decode for umt5 teachers: 5.x batch_decode inserts spurious + spaces between sentencepiece pieces; pieces must join directly (the + targets are unspaced IPA strings).""" + skip = {tok.pad_token, tok.eos_token, tok.bos_token} + return "".join(p for p in tok.convert_ids_to_tokens(ids) if p not in skip) + + @app.function( gpu="A10G", cpu=8, @@ -316,6 +328,98 @@ def metrics(model) -> dict: +@app.function( + gpu="A10G", + cpu=8, + memory=32 * 1024, + timeout=2 * 3600, + volumes={ + "/datasets": DATASETS, + "/checkpoints": CHECKPOINTS, + "/secryst-checkpoints": SECRYST_CHECKPOINTS, + "/secryst-datasets": SECRYST_DATASETS, + "/persian-checkpoints": PERSIAN_CHECKPOINTS, + }, +) +def evaluate_per(spec_id: str, limit: int = 0) -> dict: + """PER of teacher and student on the same g2p test split, replicating + the source-model harness exactly (train_thai_combined.py::evaluate): + held-out test file, beam-4 decode, corpus-level PER = + total_ed / total_gold over whitespace tokens. The student gate is + teacher_per + 5pp (docs/DISTILL-SOURCE-PROMPT.md).""" + import json + from pathlib import Path + + import torch + from transformers import AutoModelForSeq2SeqLM, AutoTokenizer + + spec = SPECS[spec_id] + teacher_vol = spec.get("teacher_volume", "rababa") + vol_map = { + "rababa": "/checkpoints", + "secryst": "/secryst-checkpoints", + "persian": "/persian-checkpoints", + } + data_vol = "/secryst-datasets" if teacher_vol == "secryst" else "/datasets" + teacher_path = Path(vol_map[teacher_vol]) / spec["teacher"] + student_path = Path(vol_map[teacher_vol]) / spec["out"] / "best" + test_path = Path(data_vol) / spec.get("eval_test", spec["test"]) + + teacher_tok = AutoTokenizer.from_pretrained(str(teacher_path)) + teacher = AutoModelForSeq2SeqLM.from_pretrained(str(teacher_path)).to("cuda").eval() + student_tok = AutoTokenizer.from_pretrained("google/byt5-small") + student = ( + AutoModelForSeq2SeqLM.from_pretrained(str(student_path)).to("cuda").eval() + ) + + pairs = [] + for line in test_path.read_text(encoding="utf-8").splitlines(): + if line.strip(): + row = json.loads(line) + pairs.append((row["src"], row["tgt"])) + if limit: + pairs = pairs[:limit] + print(f"[{spec_id}] eval pairs: {len(pairs)} from {test_path}", flush=True) + + def beam(tok, model, batch: list[str], max_len: int = 256) -> list[str]: + enc = tok(batch, return_tensors="pt", padding=True, truncation=True, + max_length=max_len).to("cuda") + with torch.no_grad(): + out = model.generate(**enc, max_new_tokens=max_len, num_beams=4) + return tok.batch_decode(out, skip_special_tokens=True) + + def per(model, tok, debug_name: str) -> dict: + total_ed = total_gold = exact = n = 0 + for start in range(0, len(pairs), 32): + batch = pairs[start : start + 32] + preds = beam(tok, model, [src for src, _ in batch]) + for (_, gold), pred in zip(batch, preds, strict=True): + e = _edit_distance(pred.strip().split(), gold.strip().split()) + total_ed += e + total_gold += max(1, len(gold.split())) + exact += e == 0 + n += 1 + if start == 0: + for (src, gold), pred in zip(batch[:3], preds[:3], strict=True): + print( + f"[{debug_name}] src={src!r}\n gold={gold!r}\n pred={pred!r}", + flush=True, + ) + return { + "per": round(100 * total_ed / max(1, total_gold), 2), + "exact_match": round(100 * exact / max(1, n), 2), + "n": n, + } + + result = { + "teacher": per(teacher, teacher_tok, "teacher"), + "student": per(student, student_tok, "student"), + } + result["gate_delta"] = round(result["student"]["per"] - result["teacher"]["per"], 2) + result["gate_pass"] = result["gate_delta"] <= 5.0 + return result + + @app.function( gpu="A10G", cpu=8, @@ -358,7 +462,6 @@ def distill_sequence(spec_id: str, epochs: int = 3) -> dict: data_vol = "/secryst-datasets" if teacher_vol == "secryst" else "/datasets" train_path = Path(data_vol) / spec["train"] - val_path = Path(data_vol) / spec["val"] # Teacher: use its OWN tokenizer (sentencepiece for umt5) teacher_tok = AutoTokenizer.from_pretrained(str(teacher_path)) @@ -425,7 +528,7 @@ def collate(batch): ).to("cuda") with torch.no_grad(): out = teacher.generate(**enc, max_new_tokens=384, num_beams=1) - preds = teacher_tok.batch_decode(out, skip_special_tokens=True) + preds = [decode_joined(teacher_tok, o) for o in out] for (src, _), pred in zip(batch, preds, strict=True): fh.write( json.dumps( @@ -471,7 +574,7 @@ def __getitem__(self, i): save_every = 500 step = 0 - for epoch in range(epochs): + for _ in range(epochs): for ids, am, labels in train_loader: ids, am, labels = ids.to("cuda"), am.to("cuda"), labels.to("cuda") loss = student(input_ids=ids, attention_mask=am, labels=labels).loss @@ -515,3 +618,8 @@ def main(spec: str = "heb-diac-small", epochs: int = 3) -> None: @app.local_entrypoint() def eval_main(spec: str = "heb-diac-small", limit: int = 0) -> None: print(evaluate.remote(spec, limit)) + + +@app.local_entrypoint() +def eval_per(spec: str = "tha-g2p-small", limit: int = 0) -> None: + print(evaluate_per.remote(spec, limit)) diff --git a/src/gpu/modal_export.py b/src/gpu/modal_export.py index 145356a..c933e85 100644 --- a/src/gpu/modal_export.py +++ b/src/gpu/modal_export.py @@ -93,6 +93,15 @@ "test_data": "urdu-diacrit/test.jsonl", "probe": "اردو", }, + "tha-g2p-small": { + "volume": "/volumes/secryst-checkpoints", + "checkpoint": "secryst_thai_g2p_distill_small/run-002/best", + "metadata": "models/tha-g2p-small/tha-g2p-small-1.0.metadata.yaml", + "readme": "models/tha-g2p-small/tha-g2p-small-1.0.README.md", + "test_volume": "/datasets/secryst", + "test_data": "thai-ipa/test.jsonl", + "probe": "สวัสดี", + }, } app = modal.App("interscript-ml-export", image=IMAGE) diff --git a/src/gpu/modal_teacher_thai.py b/src/gpu/modal_teacher_thai.py new file mode 100644 index 0000000..bbb11d6 --- /dev/null +++ b/src/gpu/modal_teacher_thai.py @@ -0,0 +1,215 @@ +"""Recover the Thai umt5 teacher (WO07 prerequisite). + +The secryst umt5 checkpoints on secryst-checkpoints were saved under +transformers 5.15.0, which dropped the trained (untied) umt5 lm_head — +every saved artifact degenerates at inference (verified 2026-08-18; +published PERs 2.32/3.24% were live-eval numbers, not reproducible from +the saved files). This script re-finetunes the same recipe under +transformers 5.14.1 (the version all working ByT5 exports used): + + base B-K/umt5-thai-g2p-v2-0.5k (HF hub, loads correctly) + data thai-ipa-expanded/train.jsonl (the 60K Kaikki+epitran mix) + eval thai-ipa/test.jsonl, beam-4, joined-piece decode, corpus PER + + modal run --detach src/gpu/modal_teacher_thai.py::main +""" + +from __future__ import annotations + +from pathlib import Path + +import modal + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent + +IMAGE = ( + modal.Image.debian_slim(python_version="3.11") + .pip_install( + "torch==2.12.1", + "transformers==5.14.1", + "pyyaml>=6.0", + "numpy>=1.26", + ) + .add_local_dir(str(REPO_ROOT), "/root/ml-models", copy=True) + .workdir("/root/ml-models") +) + +CKPTS = modal.Volume.from_name("secryst-checkpoints") +DATA = modal.Volume.from_name("secryst-datasets") + +BASE = "B-K/umt5-thai-g2p-v2-0.5k" +OUT = "secryst_thai_ipa_teacher_recovery/run-002" + +app = modal.App("tha-teacher-recovery", image=IMAGE) + + +def _decode_joined(tok, ids) -> str: + pieces = tok.convert_ids_to_tokens(ids) + skip = {tok.pad_token, tok.eos_token, tok.bos_token} + return "".join(p for p in pieces if p not in skip) + + +@app.function( + gpu="A10G", + cpu=8, + memory=32 * 1024, + timeout=6 * 3600, + volumes={"/ckpts": CKPTS, "/datasets": DATA}, +) +def train(epochs: int = 3, lr: float = 3e-4, batch: int = 16) -> dict: + import json + + import torch + from torch.utils.data import DataLoader, Dataset + from transformers import ( + AutoModelForSeq2SeqLM, + AutoTokenizer, + get_cosine_schedule_with_warmup, + ) + + device = "cuda" + out_root = Path("/ckpts") / OUT + out_root.mkdir(parents=True, exist_ok=True) + + tok = AutoTokenizer.from_pretrained(BASE) + model = AutoModelForSeq2SeqLM.from_pretrained(BASE).to(device) + model.train() + + class Pairs(Dataset): + def __init__(self, paths: list[Path], max_len: int = 384): + self.rows = [] + for path in paths: + for line in path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + row = json.loads(line) + s, t = (row.get("src") or "").strip(), (row.get("tgt") or "").strip() + if s and t: + self.rows.append((s, t)) + + def __len__(self): + return len(self.rows) + + def __getitem__(self, i): + return self.rows[i] + + def collate(batch): + src = tok([s for s, _ in batch], padding=True, truncation=True, + max_length=384, return_tensors="pt") + labels = tok([t for _, t in batch], padding=True, truncation=True, + max_length=384, return_tensors="pt").input_ids + labels[labels == tok.pad_token_id] = -100 + return src.input_ids, src.attention_mask, labels + + # secryst's exact combined recipe (train_thai_combined.py): + # Kaikki 9.7K + epitran-augmented 50K + train_paths = [ + Path("/datasets/thai-ipa/train.jsonl"), + Path("/datasets/thai-ipa/augmented_epitran.jsonl"), + ] + loader = DataLoader(Pairs(train_paths), batch_size=batch, shuffle=True, + collate_fn=collate, num_workers=2, drop_last=True) + total_steps = len(loader) * epochs + print(f"pairs={len(loader.dataset)} steps={total_steps}", flush=True) + + optimizer = torch.optim.AdamW(model.parameters(), lr=lr) + scheduler = get_cosine_schedule_with_warmup(optimizer, total_steps // 20, total_steps) + + start_step = 0 + ckpts = sorted(out_root.glob("step-*"), key=lambda p: int(p.name.split("-")[1])) + if ckpts: + model.load_state_dict(torch.load(ckpts[-1] / "model.pt", map_location=device, + weights_only=True)) + optimizer.load_state_dict(torch.load(ckpts[-1] / "optim.pt", map_location=device, + weights_only=True)) + start_step = int(ckpts[-1].name.split("-")[1]) + for _ in range(start_step): + scheduler.step() + print(f"[resume] from step-{start_step}", flush=True) + + step = start_step + while step < total_steps: + for ids, am, labels in loader: + if step >= total_steps: + break + ids, am, labels = ids.to(device), am.to(device), labels.to(device) + loss = model(input_ids=ids, attention_mask=am, labels=labels).loss + loss.backward() + torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) + optimizer.step() + scheduler.step() + optimizer.zero_grad() + step += 1 + if step % 100 == 0: + print(f"[step {step}/{total_steps}] loss={float(loss):.4f}", flush=True) + if step % 1000 == 0: + ck = out_root / f"step-{step}" + ck.mkdir(exist_ok=True) + torch.save(model.state_dict(), ck / "model.pt") + torch.save(optimizer.state_dict(), ck / "optim.pt") + CKPTS.commit() + + best = out_root / "best" + best.mkdir(exist_ok=True) + model.save_pretrained(str(best)) + tok.save_pretrained(str(best)) + CKPTS.commit() + return {"out": OUT, "steps": step} + + +@app.function( + gpu="A10G", + cpu=4, + memory=16 * 1024, + timeout=2 * 3600, + volumes={"/ckpts": CKPTS, "/datasets": DATA}, +) +def evaluate(limit: int = 0) -> dict: + import json + + import torch + from transformers import AutoModelForSeq2SeqLM, AutoTokenizer + + ckpt = Path("/ckpts") / OUT / "best" + tok = AutoTokenizer.from_pretrained(str(ckpt)) + model = AutoModelForSeq2SeqLM.from_pretrained(str(ckpt)).to("cuda").eval() + + pairs = [] + for line in (Path("/datasets/thai-ipa/test.jsonl")).read_text(encoding="utf-8").splitlines(): + if line.strip(): + row = json.loads(line) + pairs.append((row["src"], row["tgt"])) + if limit: + pairs = pairs[:limit] + + def ed(a, b): + prev = list(range(len(b) + 1)) + for i, ai in enumerate(a, 1): + curr = [i] + for j, bj in enumerate(b, 1): + curr.append(min(prev[j] + 1, curr[j - 1] + 1, prev[j - 1] + (ai != bj))) + prev = curr + return prev[-1] + + total_ed = total_gold = exact = 0 + with torch.no_grad(): + for start in range(0, len(pairs), 32): + batch = pairs[start : start + 32] + enc = tok([s for s, _ in batch], return_tensors="pt", padding=True, + truncation=True, max_length=256).to("cuda") + out = model.generate(**enc, max_new_tokens=256, num_beams=4) + preds = [_decode_joined(tok, o) for o in out] + for (_, gold), pred in zip(batch, preds, strict=True): + e = ed(list(pred.strip()), list(gold.strip())) + total_ed += e + total_gold += max(1, len(gold)) + exact += e == 0 + per = round(100 * total_ed / max(1, total_gold), 2) + print(f"teacher PER={per} exact={round(100 * exact / max(1, len(pairs)), 2)} n={len(pairs)}", flush=True) + return {"per": per, "n": len(pairs)} + + +@app.local_entrypoint() +def main(epochs: int = 3) -> None: + print(train.remote(epochs=epochs)) + print(evaluate.remote()) From aab40dc82d39b747ba4c37ef985b36081cdbcf24 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Tue, 18 Aug 2026 17:05:45 +0800 Subject: [PATCH 2/4] fix: import order for ruff --- scripts/generate_release_notes.py | 1 - scripts/publish_model.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/generate_release_notes.py b/scripts/generate_release_notes.py index c2d88a2..962511c 100644 --- a/scripts/generate_release_notes.py +++ b/scripts/generate_release_notes.py @@ -16,7 +16,6 @@ import sys from pathlib import Path - TEMPLATE = """\ ## {task} v{version} diff --git a/scripts/publish_model.py b/scripts/publish_model.py index 1c8b826..e7465f3 100644 --- a/scripts/publish_model.py +++ b/scripts/publish_model.py @@ -30,8 +30,8 @@ sys.path.insert(0, str(REPO_ROOT / "src")) sys.path.insert(0, str(REPO_ROOT / "scripts")) -from split_release import split as split_zip # noqa: E402 from imf.validator import validate_zip # noqa: E402 +from split_release import split as split_zip # noqa: E402 # GitHub hard-caps release assets at 2,147,483,648 bytes; split well below. SPLIT_THRESHOLD = 2_000_000_000 From 9e8d78b90f30da265fcdae517276e78cda51759a Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Tue, 18 Aug 2026 17:06:06 +0800 Subject: [PATCH 3/4] fix: line length --- src/gpu/modal_teacher_thai.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/gpu/modal_teacher_thai.py b/src/gpu/modal_teacher_thai.py index bbb11d6..9bba4e3 100644 --- a/src/gpu/modal_teacher_thai.py +++ b/src/gpu/modal_teacher_thai.py @@ -205,7 +205,8 @@ def ed(a, b): total_gold += max(1, len(gold)) exact += e == 0 per = round(100 * total_ed / max(1, total_gold), 2) - print(f"teacher PER={per} exact={round(100 * exact / max(1, len(pairs)), 2)} n={len(pairs)}", flush=True) + exact_pct = round(100 * exact / max(1, len(pairs)), 2) + print(f"teacher PER={per} exact={exact_pct} n={len(pairs)}", flush=True) return {"per": per, "n": len(pairs)} From 3502dcfc4251c3381dfb6bcdfe6b9aeb3140554b Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Tue, 18 Aug 2026 17:08:26 +0800 Subject: [PATCH 4/4] fix(publish): metadata under family dir models//, not id-named dir --- scripts/publish_model.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/publish_model.py b/scripts/publish_model.py index e7465f3..72515ef 100644 --- a/scripts/publish_model.py +++ b/scripts/publish_model.py @@ -210,10 +210,12 @@ def main() -> None: try: wt_models = worktree / "models.yaml" wt_repo = str(worktree) + # models//.metadata.yaml, e.g. models/heb-diac/heb-diac-1.0... + family = args.model_id.rsplit("-", 1)[0] upsert_models_yaml(wt_models, args.model_id, entry_block(args.model_id, meta, args.zip.name, whole_sha, size, assets, args.repo, tag)) - model_dir = worktree / "models" / args.model_id + model_dir = worktree / "models" / family model_dir.mkdir(parents=True, exist_ok=True) (model_dir / f"{args.model_id}.metadata.yaml").write_text( yaml.safe_dump(meta, sort_keys=False, allow_unicode=True), encoding="utf-8") @@ -221,7 +223,7 @@ def main() -> None: def git_wt(*cmd: str): return run(["git", "-C", wt_repo, *cmd], capture_output=True, text=True) - git_wt("add", "models.yaml", f"models/{args.model_id}/{args.model_id}.metadata.yaml") + git_wt("add", "models.yaml", f"models/{family}/{args.model_id}.metadata.yaml") staged = git_wt("diff", "--cached", "--name-only").stdout.split() if staged: git_wt("commit", "-m", f"release: {args.model_id} "