submit: retry Hub commits on 429/5xx + persistent status panel

Wrap every Space-side commit (zip upload, results.jsonl RMW, report +
gallery uploads, shard cleanup) in a backoff-with-jitter retry that
honors Retry-After, so a rate-limited submissions repo delays rather
than fails a submit. Make handle_submit a generator streaming a
durable status panel (validating -> uploading -> queued, or the
rejection reason) instead of relying on transient toasts.

Co-authored-by: Cursor <cursoragent@cursor.com>

app.py

@@ -123,6 +123,10 @@ VALIDATION_GUIDELINES_MD = f"""Submissions appear on the **Unvalidated** table t
 Full policy: [`docs/benchmark/validation.md`]({VALIDATION_DOC_URL})."""
 
 DETAIL_PLACEHOLDER = "_Click a row above for details._"
+SUBMIT_STATUS_IDLE = (
+    "_Log in, attach a zip, and click **Submit**. Progress and any "
+    "errors appear here._"
+)
 
 
 def _has(value) -> bool:
@@ -784,11 +788,19 @@ to publish the resulting row on the public leaderboard.
         # Starts disabled; the `blocks.load` handler below flips it
         # to interactive when an OAuthProfile is present.
         submit_btn = gr.Button("Submit", variant="primary", interactive=False)
-        # No static markdown output: handle_submit surfaces every
-        # status update via gr.Info / gr.Error toasts. The handler
-        # also reads `gr.OAuthProfile` implicitly via its parameter
-        # type annotation (Gradio's dependency-injection convention).
-        submit_btn.click(fn=handle_submit, inputs=[zip_in])
+        # Persistent status panel. handle_submit is a generator that
+        # streams stage updates (validating -> uploading/queuing ->
+        # queued) and any rejection reason here, so the outcome
+        # survives instead of vanishing with a transient toast. The
+        # handler also reads `gr.OAuthProfile` implicitly via its
+        # parameter type annotation (Gradio's dependency-injection
+        # convention).
+        submit_status = gr.Markdown(value=SUBMIT_STATUS_IDLE)
+        submit_btn.click(
+            fn=handle_submit,
+            inputs=[zip_in],
+            outputs=[submit_status],
+        )
 
     with gr.Tab("About"):
         gr.Markdown(ABOUT_MD)

submit.py

@@ -88,6 +88,7 @@ import hashlib
 import json
 import logging
 import os
+import random
 import re
 import shutil
 import tempfile
@@ -111,7 +112,7 @@ from huggingface_hub import (
     run_job,
     snapshot_download,
 )
-from huggingface_hub.errors import EntryNotFoundError
+from huggingface_hub.errors import EntryNotFoundError, HfHubHTTPError
 
 from leaderboard import HF_DATA_REPO, HF_ORG, HF_SUBMISSIONS_REPO
 
@@ -138,6 +139,18 @@ GALLERY_THUMB_VIEW = "iso"
 DATA_REV_SHORT_LEN = 12
 FAILURE_REASON_MAX_CHARS = 200
 SHA256_BLOCK_SIZE = 64 * 1024
+
+# Hub-write retry policy. HF rate-limits commits per repo, so a burst
+# of concurrent commits (e.g. bulk baseline seeding writing into this
+# same dataset) can 429 an otherwise-valid submit at the commit step.
+# Retry transient statuses with exponential backoff + jitter, honoring
+# a server Retry-After, up to a wall-clock cap. Applied to every
+# Space-side commit so a busy repo delays a submit instead of failing
+# it.
+_HUB_RETRY_STATUSES = frozenset({429, 500, 502, 503, 504})
+HUB_RETRY_MAX_SECONDS = 120
+HUB_RETRY_BASE_DELAY_SECONDS = 2.0
+HUB_RETRY_MAX_DELAY_SECONDS = 20.0
 STUCK_PENDING_THRESHOLD_SECONDS = 30 * 60
 SUBMITTED_AT_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
 STUCK_PENDING_REASON = "evaluation interrupted by Space restart"
@@ -214,49 +227,109 @@ class _HubWriteError(Exception):
     """Raised when a Hub upload fails after validation succeeded."""
 
 
+def _retry_after_seconds(error: HfHubHTTPError) -> float | None:
+    """Parse a ``Retry-After`` header (seconds form) off a Hub error, if any."""
+    response = getattr(error, "response", None)
+    if response is None:
+        return None
+    raw = response.headers.get("Retry-After")
+    if not raw:
+        return None
+    try:
+        return float(raw)
+    except (TypeError, ValueError):
+        return None
+
+
+def _with_hub_retries(fn, *, what: str):
+    """Run *fn* (a Hub commit) retrying transient HTTP errors with backoff.
+
+    Retries only the statuses in :data:`_HUB_RETRY_STATUSES` (rate
+    limits + transient 5xx); any other error (auth, validation, a
+    ``LookupError`` from a mutate closure) propagates immediately.
+    Backoff is exponential with jitter, clamped to
+    :data:`HUB_RETRY_MAX_DELAY_SECONDS`, never sleeps past the
+    :data:`HUB_RETRY_MAX_SECONDS` wall cap, and honors a server
+    ``Retry-After`` when present. *fn* must be idempotent across calls
+    -- every caller here re-reads the remote state inside *fn* before
+    committing, so a retried commit can't double-apply.
+    """
+    deadline = time.monotonic() + HUB_RETRY_MAX_SECONDS
+    attempt = 0
+    while True:
+        attempt += 1
+        try:
+            return fn()
+        except HfHubHTTPError as e:
+            status = getattr(getattr(e, "response", None), "status_code", None)
+            now = time.monotonic()
+            if status not in _HUB_RETRY_STATUSES or now >= deadline:
+                raise
+            delay = min(
+                HUB_RETRY_MAX_DELAY_SECONDS,
+                HUB_RETRY_BASE_DELAY_SECONDS * (2 ** (attempt - 1)),
+            )
+            delay = delay / 2 + random.uniform(0, delay / 2)
+            retry_after = _retry_after_seconds(e)
+            if retry_after is not None:
+                delay = max(delay, retry_after)
+            delay = min(delay, max(0.0, deadline - now))
+            logger.warning(
+                "Hub %s got HTTP %s; retry %d in %.1fs (cap %ds)",
+                what, status, attempt, delay, HUB_RETRY_MAX_SECONDS,
+            )
+            time.sleep(delay)
+
+
+def _submit_status(state: str, message: str) -> str:
+    """Markdown for the persistent submit-status panel.
+
+    The panel is the durable counterpart to the transient ``gr.Info`` /
+    ``gr.Error`` toasts: a submitter always sees the current stage and
+    any rejection reason without having to catch an ephemeral toast.
+    *state* picks the leading glyph (``working`` / ``queued`` /
+    ``error``).
+    """
+    glyph = {"working": "⏳", "queued": "✅", "error": "❌"}.get(state, "•")
+    return f"{glyph} {message}"
+
+
 def handle_submit(
     zip_file,
     profile: gr.OAuthProfile | None,
-) -> None:
-    """Validate a submission upload; surface progress + outcome via toasts.
+):
+    """Validate + queue a submission, streaming status to a panel.
 
     Requires the user to be logged in via ``gr.LoginButton`` so the
     row's ``hf_username`` is the canonical HF identity (not a
     free-text claim). The submit button in :mod:`app` is disabled
-    until login completes; this function also raises ``gr.Error``
-    defensively if it's called without a profile so a UI mishap
-    can't write an anonymous row.
-
-    Side-effect-only (returns ``None``): in-flight status surfaces via
-    ``gr.Info`` toasts (validating on click, queued after the worker
-    spawns) and ``gr.Error`` toasts for rejections. Rejection paths
-    raise ``gr.Error``, which Gradio surfaces as a red toast and
-    aborts the handler; the outer ``try/finally`` still runs to clean
-    up the temp dir.
-
-    Toasts on the happy path:
-
-    1. ``"Validating submission..."`` on click, with a wall-time hint
-       so the user knows the post-queue eval takes ~1 minute.
-    2. ``"Submission <id> queued ..."`` once the row + zip are on the
-       Hub and the worker has been spawned.
-
-    On rejection (login-missing, form-level, validation gate, dedup,
-    or Hub write), a single ``gr.Error`` toast carries the message;
-    no second toast.
+    until login completes; this generator also rejects defensively if
+    it's called without a profile so a UI mishap can't write an
+    anonymous row.
+
+    Generator: each ``yield`` is a Markdown string pushed to a
+    persistent status panel, the durable counterpart to the transient
+    toasts. Happy-path stages: validating -> uploading/queuing ->
+    queued. Every rejection (login-missing, form-level, validation
+    gate, dedup, Hub write) yields a final error panel **and** raises
+    ``gr.Error`` for a toast; the outer ``try/finally`` still runs to
+    clean up the temp dir. The Hub writes inside ride out transient
+    rate limits via :func:`_with_hub_retries`, so a busy submissions
+    repo delays rather than fails the submit.
     """
     if profile is None:
-        raise gr.Error("Please log in via the HF button before submitting.")
+        msg = "Please log in via the HF button before submitting."
+        yield _submit_status("error", msg)
+        raise gr.Error(msg)
     form_err = _validate_form(zip_file)
     if form_err is not None:
+        yield _submit_status("error", form_err)
         raise gr.Error(form_err)
 
-    # Immediate ack on click. The eval itself (after queuing) is the
-    # slow part, so hint at the wall time up front so the user knows
-    # what to expect instead of waiting on a silent button.
-    gr.Info(
-        "Validating submission... evaluation usually takes "
-        "~1 minute after queuing."
+    yield _submit_status(
+        "working",
+        "Validating submission (unpacking the zip, checking the fixture set "
+        "and STEP files)…",
     )
 
     zip_path = Path(zip_file.name)
@@ -276,7 +349,9 @@ def handle_submit(
             fixture_names = _validate_fixture_set(run_dir)
             _validate_steps_parseable(run_dir, fixture_names)
         except _ValidationError as e:
-            raise gr.Error(f"Submission rejected: {e}")
+            msg = f"Submission rejected: {e}"
+            yield _submit_status("error", msg)
+            raise gr.Error(msg)
 
         # Dedup gate: hash the raw zip bytes and reject if an existing
         # row carries the same hash. Runs after validation so a clearly
@@ -284,15 +359,23 @@ def handle_submit(
         zip_sha256 = _compute_sha256(zip_path)
         existing_id = _find_existing_submission_by_sha256(zip_sha256)
         if existing_id is not None:
-            raise gr.Error(
+            msg = (
                 f"This zip's contents are identical to an existing "
                 f"submission ({existing_id}). Resubmit only after changing "
                 f"at least one byte of the upload."
             )
+            yield _submit_status("error", msg)
+            raise gr.Error(msg)
 
         submission_id = _mint_submission_id(
             meta["submitter_name"], meta["submission_name"]
         )
+        yield _submit_status(
+            "working",
+            f"Uploading `{submission_id}` ({len(fixture_names)} fixtures) and "
+            f"queuing the evaluation… (this can take a moment, and retries "
+            f"automatically if the Hub is busy).",
+        )
         try:
             blob_url = _upload_submission_zip(submission_id, zip_path)
             row = _build_pending_row(
@@ -301,14 +384,17 @@ def handle_submit(
             )
             _append_pending_row(row)
         except _HubWriteError as e:
-            raise gr.Error(f"Submission rejected: {e}")
+            msg = f"Submission rejected: {e}"
+            yield _submit_status("error", msg)
+            raise gr.Error(msg)
 
         _spawn_worker(submission_id, blob_url, sorted(fixture_names))
-        gr.Info(
-            f"Submission {submission_id} queued for evaluation "
-            f"({len(fixture_names)} fixtures). The eval runs on an "
-            f"HF Jobs GPU; the row flips to completed when the job "
-            f"finishes (typically 1-3 minutes)."
+        yield _submit_status(
+            "queued",
+            f"Submission `{submission_id}` queued ({len(fixture_names)} "
+            f"fixtures). The eval runs on an HF Jobs GPU; your row appears on "
+            f"the **Unvalidated** leaderboard and flips to completed when the "
+            f"job finishes (typically 1–3 minutes).",
         )
     finally:
         shutil.rmtree(tmp, ignore_errors=True)
@@ -509,12 +595,15 @@ def _upload_submission_zip(submission_id: str, zip_path: Path) -> str:
     """
     repo_path = f"{SUBMISSIONS_DIR}/{submission_id}.zip"
     try:
-        _HF_API.upload_file(
-            path_or_fileobj=str(zip_path),
-            path_in_repo=repo_path,
-            repo_id=HF_SUBMISSIONS_REPO,
-            repo_type="dataset",
-            commit_message=f"add submission zip for {submission_id}",
+        _with_hub_retries(
+            lambda: _HF_API.upload_file(
+                path_or_fileobj=str(zip_path),
+                path_in_repo=repo_path,
+                repo_id=HF_SUBMISSIONS_REPO,
+                repo_type="dataset",
+                commit_message=f"add submission zip for {submission_id}",
+            ),
+            what="submission-zip upload",
         )
     except Exception as e:  # noqa: BLE001 - Hub API surface is broad
         logger.exception("Failed to upload submission zip %s", submission_id)
@@ -667,8 +756,14 @@ def _hub_rmw_results(
     The lock is held only for the read-modify-write cycle (~1-2s),
     never for eval time. Concurrent submitters serialise here, not
     in the eval pipeline. Treats a missing file as the empty list.
+
+    The whole download->mutate->upload cycle is retried as a unit on a
+    transient Hub error (:func:`_with_hub_retries`): re-reading the
+    current file each attempt keeps the mutation idempotent and also
+    folds in any concurrent change, so a rate-limited commit waits and
+    retries instead of failing the caller.
     """
-    with _HUB_LOCK:
+    def _download_mutate_upload() -> None:
         existing = _download_results_jsonl()
         rows: list[dict[str, Any]] = [
             json.loads(line) for line in existing.splitlines() if line.strip()
@@ -687,6 +782,9 @@ def _hub_rmw_results(
             commit_message=commit_message,
         )
 
+    with _HUB_LOCK:
+        _with_hub_retries(_download_mutate_upload, what="results.jsonl write")
+
 
 def _download_results_jsonl() -> str:
     """Fetch the current ``results.jsonl`` body as text, or ``""`` if absent."""
@@ -1239,21 +1337,27 @@ def _upload_reports(
     artifacts land at the exact paths the leaderboard + the row-flip
     expect. Uses the process ``HfApi`` (Space HF_TOKEN env).
     """
-    _HF_API.upload_file(
-        path_or_fileobj=str(html_path),
-        path_in_repo=f"{REPORTS_DIR}/{submission_id}.html",
-        repo_id=HF_SUBMISSIONS_REPO,
-        repo_type="dataset",
-        commit_message=f"add merged HTML report for {submission_id}",
+    _with_hub_retries(
+        lambda: _HF_API.upload_file(
+            path_or_fileobj=str(html_path),
+            path_in_repo=f"{REPORTS_DIR}/{submission_id}.html",
+            repo_id=HF_SUBMISSIONS_REPO,
+            repo_type="dataset",
+            commit_message=f"add merged HTML report for {submission_id}",
+        ),
+        what="report-html upload",
     )
-    _HF_API.upload_file(
-        path_or_fileobj=json.dumps(
-            report_json, ensure_ascii=False, indent=2,
-        ).encode("utf-8"),
-        path_in_repo=f"{REPORTS_DIR}/{submission_id}.json",
-        repo_id=HF_SUBMISSIONS_REPO,
-        repo_type="dataset",
-        commit_message=f"add merged JSON report for {submission_id}",
+    _with_hub_retries(
+        lambda: _HF_API.upload_file(
+            path_or_fileobj=json.dumps(
+                report_json, ensure_ascii=False, indent=2,
+            ).encode("utf-8"),
+            path_in_repo=f"{REPORTS_DIR}/{submission_id}.json",
+            repo_id=HF_SUBMISSIONS_REPO,
+            repo_type="dataset",
+            commit_message=f"add merged JSON report for {submission_id}",
+        ),
+        what="report-json upload",
     )
     logger.info("Uploaded merged reports/%s.{html,json}", submission_id)
 
@@ -1280,16 +1384,21 @@ def _upload_gallery_renders_from_dir(
         logger.info("No gallery renders to upload for %s", submission_id)
         return
     for iso_png, fixture_name in staged:
-        _HF_API.upload_file(
-            path_or_fileobj=str(iso_png),
-            path_in_repo=(
-                f"{RENDERS_DIR}/{submission_id}/{fixture_name}.png"
-            ),
-            repo_id=HF_SUBMISSIONS_REPO,
-            repo_type="dataset",
-            commit_message=(
-                f"add gallery render {fixture_name} for {submission_id}"
+        _with_hub_retries(
+            lambda iso_png=iso_png, fixture_name=fixture_name: (
+                _HF_API.upload_file(
+                    path_or_fileobj=str(iso_png),
+                    path_in_repo=(
+                        f"{RENDERS_DIR}/{submission_id}/{fixture_name}.png"
+                    ),
+                    repo_id=HF_SUBMISSIONS_REPO,
+                    repo_type="dataset",
+                    commit_message=(
+                        f"add gallery render {fixture_name} for {submission_id}"
+                    ),
+                )
             ),
+            what="gallery-render upload",
         )
     logger.info(
         "Uploaded %d gallery render(s) under %s/%s/",
@@ -1306,11 +1415,14 @@ def _cleanup_shard_artifacts(submission_id: str) -> None:
     submission.
     """
     try:
-        _HF_API.delete_folder(
-            path_in_repo=f"{REPORTS_DIR}/{submission_id}/{SHARDS_SUBDIR}",
-            repo_id=HF_SUBMISSIONS_REPO,
-            repo_type="dataset",
-            commit_message=f"clean up eval shards for {submission_id}",
+        _with_hub_retries(
+            lambda: _HF_API.delete_folder(
+                path_in_repo=f"{REPORTS_DIR}/{submission_id}/{SHARDS_SUBDIR}",
+                repo_id=HF_SUBMISSIONS_REPO,
+                repo_type="dataset",
+                commit_message=f"clean up eval shards for {submission_id}",
+            ),
+            what="shard cleanup",
         )
         logger.info("Cleaned up shard artifacts for %s", submission_id)
     except Exception as e:  # noqa: BLE001 - cleanup is best-effort

tests/test_submit.py

@@ -12,10 +12,92 @@ network traffic.
 from __future__ import annotations
 
 from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
 
 import submit
 
 
+def _hub_http_error(status: int, headers: dict | None = None) -> submit.HfHubHTTPError:
+    """An ``HfHubHTTPError`` with a minimal response carrying *status*.
+
+    Built without going through the real Hub: construct the exception
+    with no response, then attach a stand-in so ``_with_hub_retries``
+    can read ``response.status_code`` / ``response.headers``.
+    """
+    err = submit.HfHubHTTPError(f"HTTP {status}")
+    err.response = SimpleNamespace(status_code=status, headers=headers or {})
+    return err
+
+
+def test_with_hub_retries_recovers_after_transient(monkeypatch):
+    """A 429 (then a 503) is retried; the eventual success is returned."""
+    monkeypatch.setattr(submit.time, "sleep", lambda *_: None)
+    statuses = iter([429, 503])
+    calls = {"n": 0}
+
+    def flaky():
+        calls["n"] += 1
+        status = next(statuses, None)
+        if status is not None:
+            raise _hub_http_error(status)
+        return "ok"
+
+    assert submit._with_hub_retries(flaky, what="test") == "ok"
+    assert calls["n"] == 3
+
+
+def test_with_hub_retries_reraises_non_retryable(monkeypatch):
+    """A 403 is not in the retry set, so it propagates on the first try."""
+    monkeypatch.setattr(submit.time, "sleep", lambda *_: None)
+    calls = {"n": 0}
+
+    def forbidden():
+        calls["n"] += 1
+        raise _hub_http_error(403)
+
+    with pytest.raises(submit.HfHubHTTPError):
+        submit._with_hub_retries(forbidden, what="test")
+    assert calls["n"] == 1
+
+
+def test_with_hub_retries_gives_up_after_wall_cap(monkeypatch):
+    """Past the wall cap, a persistent 429 stops being retried and raises."""
+    monkeypatch.setattr(submit.time, "sleep", lambda *_: None)
+    # Force the deadline check to trip after the first failure: the
+    # second monotonic() read (post-failure) already exceeds the cap.
+    ticks = iter([0.0, submit.HUB_RETRY_MAX_SECONDS + 1])
+    monkeypatch.setattr(
+        submit.time, "monotonic", lambda: next(ticks, submit.HUB_RETRY_MAX_SECONDS + 1)
+    )
+    calls = {"n": 0}
+
+    def always_429():
+        calls["n"] += 1
+        raise _hub_http_error(429)
+
+    with pytest.raises(submit.HfHubHTTPError):
+        submit._with_hub_retries(always_429, what="test")
+    assert calls["n"] == 1
+
+
+def test_retry_after_header_is_honored(monkeypatch):
+    """A ``Retry-After`` seconds value sets the floor for the sleep delay."""
+    slept: list[float] = []
+    monkeypatch.setattr(submit.time, "sleep", lambda d: slept.append(d))
+    calls = {"n": 0}
+
+    def flaky():
+        calls["n"] += 1
+        if calls["n"] == 1:
+            raise _hub_http_error(429, headers={"Retry-After": "7"})
+        return "ok"
+
+    assert submit._with_hub_retries(flaky, what="test") == "ok"
+    assert slept and slept[0] >= 7.0
+
+
 def _stub_meta() -> dict:
     """Minimum meta.json shape that survives ``_load_and_validate_meta``."""
     return {