From a1b6775048a43af5dd16d6a4a1ead104d729289e Mon Sep 17 00:00:00 2001 From: JarbasAi Date: Thu, 8 Oct 2026 23:11:36 +0100 Subject: [PATCH] fix: publish the ovos-intents corpus even when the CSV export repo fails main() published the CSV repo first and let any Hub error end the script, so a 404 on the CSV repo left the ovos-intents corpus unpublished. Each repo now publishes on its own, the corpus first. Each outcome prints one line, and the script exits non-zero when any repo failed. No repo is created. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/publish_hf_dataset.py | 31 +++++++---- test/unittests/test_publish_hf_dataset.py | 63 +++++++++++++++++++++++ 2 files changed, 85 insertions(+), 9 deletions(-) diff --git a/.github/scripts/publish_hf_dataset.py b/.github/scripts/publish_hf_dataset.py index 7f93d16c..23f63a3c 100644 --- a/.github/scripts/publish_hf_dataset.py +++ b/.github/scripts/publish_hf_dataset.py @@ -18,6 +18,11 @@ The ovos-localize-intents README is left untouched: this repo carries no card template for it. +Each repo is published on its own: a failure on one is printed and does not +stop the other, and the script exits non-zero when any repo failed. The corpus +goes first because it is the one a model trains on. A repo missing on the Hub +is reported, never created. + An export that holds no CSV row or no corpus row is refused before the Hub is contacted: replacing a dataset in full from an empty tree would delete it. """ @@ -88,17 +93,25 @@ def main() -> None: "refusing to publish an empty export, which would delete the published datasets.") api = HfApi(token=token) - for repo_id, build in ((CSV_REPO_ID, csv_operations), (CORPUS_REPO_ID, corpus_operations)): - operations = build(export_dir, api.list_repo_files(repo_id=repo_id, repo_type="dataset")) - api.create_commit( - repo_id=repo_id, - repo_type="dataset", - operations=operations, - commit_message=f"chore: refresh from {repository} run {run_id}", - commit_description=run_url, - ) + failed = [] + for repo_id, build in ((CORPUS_REPO_ID, corpus_operations), (CSV_REPO_ID, csv_operations)): + try: + operations = build(export_dir, api.list_repo_files(repo_id=repo_id, repo_type="dataset")) + api.create_commit( + repo_id=repo_id, + repo_type="dataset", + operations=operations, + commit_message=f"chore: refresh from {repository} run {run_id}", + commit_description=run_url, + ) + except Exception as exc: + print(f"FAILED {repo_id}: {type(exc).__name__}: {exc}") + failed.append(repo_id) + continue added = sum(isinstance(op, CommitOperationAdd) for op in operations) print(f"published {repo_id}: {added} files added, {len(operations) - added} deleted") + if failed: + sys.exit(f"failed to publish: {', '.join(failed)}") if __name__ == "__main__": diff --git a/test/unittests/test_publish_hf_dataset.py b/test/unittests/test_publish_hf_dataset.py index 9dfae788..f3951dec 100644 --- a/test/unittests/test_publish_hf_dataset.py +++ b/test/unittests/test_publish_hf_dataset.py @@ -197,3 +197,66 @@ def test_published_csv_holds_intent_rows_only(tmp_path): ["en-US", "ovos-skill-fixture", "foo.intent", "turn on the lamp"], ["en-US", "ovos-skill-fixture", "foo.intent", "turn on the light"], ] + + +def _repository_not_found(): + from types import SimpleNamespace + + from huggingface_hub.errors import RepositoryNotFoundError + + response = SimpleNamespace(status_code=404, headers={}, text="", content=b"", json=dict, request=None) + return RepositoryNotFoundError("404 Client Error: repository not found", response=response) + + +def test_missing_csv_repo_does_not_stop_the_corpus_publish(monkeypatch, tmp_path, capsys): + """A Hub 404 on the CSV repo is reported and fails the run, after the corpus repo is published.""" + from huggingface_hub import HfApi + + module = _load_module() + committed = {} + + def list_repo_files(self, repo_id, **kwargs): + if repo_id == module.CSV_REPO_ID: + raise _repository_not_found() + return [] + + monkeypatch.setattr(HfApi, "list_repo_files", list_repo_files) + monkeypatch.setattr(HfApi, "create_commit", + lambda self, repo_id, operations, **kwargs: committed.update({repo_id: operations})) + monkeypatch.setenv("HF_TOKEN", "fake") + monkeypatch.setenv("INTENTS_EXPORT_DIR", str(_export_dir(tmp_path))) + + with pytest.raises(SystemExit) as exit_info: + module.main() + + assert exit_info.value.code not in (0, None) + assert list(committed) == [module.CORPUS_REPO_ID] + out = capsys.readouterr().out.splitlines() + assert out[0].startswith(f"published {module.CORPUS_REPO_ID}: 4 files added, 0 deleted") + assert out[1].startswith(f"FAILED {module.CSV_REPO_ID}: RepositoryNotFoundError: 404 Client Error") + + +def test_failed_corpus_repo_does_not_stop_the_csv_publish(monkeypatch, tmp_path, capsys): + from huggingface_hub import HfApi + + module = _load_module() + committed = {} + + def create_commit(self, repo_id, operations, **kwargs): + if repo_id == module.CORPUS_REPO_ID: + raise RuntimeError("boom") + committed[repo_id] = operations + + monkeypatch.setattr(HfApi, "list_repo_files", lambda self, repo_id, **kwargs: []) + monkeypatch.setattr(HfApi, "create_commit", create_commit) + monkeypatch.setenv("HF_TOKEN", "fake") + monkeypatch.setenv("INTENTS_EXPORT_DIR", str(_export_dir(tmp_path))) + + with pytest.raises(SystemExit) as exit_info: + module.main() + + assert exit_info.value.code not in (0, None) + assert list(committed) == [module.CSV_REPO_ID] + out = capsys.readouterr().out.splitlines() + assert out[0] == f"FAILED {module.CORPUS_REPO_ID}: RuntimeError: boom" + assert out[1].startswith(f"published {module.CSV_REPO_ID}:")