v0.4: surface ALL task files, not just an allowlist

The previous version hardcoded six file IDs in HarborTask / _read_task_file /
_build_file_tree (task.toml, instruction.md, solution/patch.diff,
solution/solve.sh, tests/test.sh, environment/Dockerfile). Anything else in a
task dir (e.g. tests/grader.py, environment/pull_bucket.py, helper modules)
was downloaded by snapshot_download but never shown in the UI.

Generalized:

- HarborTask gains a 'files: dict[str, str]' field — every readable text
file under the task dir, keyed by POSIX-style relative path.
- parse.py: new _discover_task_files() walks task_dir.rglob('*'), skips
binaries (UTF-8 decode failures), hidden noise (.DS_Store, .git,
__pycache__, .cache, node_modules, .venv, *_cache), and oversized files
(>512 KiB). Convenience fields (instruction_md, oracle_patch, …) kept
populated from the same dict for backwards compat with anything that
reaches in by name.
- app.py: _read_task_file becomes a dict lookup with the two special cases
preserved (task.toml uses task_toml_raw; instruction.md falls back to
the inline 'task.instruction' field).
- app.py: _build_file_tree walks task.files, groups by top-level folder,
orders Overview → top-level files → folders alphabetically → children
alphabetically. Tasks with extra files (e.g. data-agent's grader.py +
pull_bucket.py) now show every artifact faithfully.
- app.py: _EXTENSION_LANGUAGE extended for .py, .md, .json, .yaml/.yml,
.ini/.cfg, .conf, .html/.css/.js/.ts, .txt/.csv/.tsv so non-shell files
render with the right Prism token.

Backwards compatible: existing datasets (e.g. AdithyaSK/click-r2e-v082post1)
keep working — same six files surface, just now via the new path.

app.py

@@ -51,6 +51,22 @@ _EXTENSION_LANGUAGE: dict[str, str] = {
     ".patch": "python",
     ".sh": "shell",
     ".bash": "shell",
+    ".py": "python",
+    ".json": "json",
+    ".yaml": "yaml",
+    ".yml": "yaml",
+    ".md": "markdown",
+    ".markdown": "markdown",
+    ".txt": "shell",
+    ".csv": "shell",
+    ".tsv": "shell",
+    ".ini": "yaml",
+    ".cfg": "yaml",
+    ".conf": "shell",
+    ".html": "html",
+    ".css": "css",
+    ".js": "javascript",
+    ".ts": "typescript",
 }
 
 
@@ -63,69 +79,74 @@ def _file_language(filename: str) -> str:
 
 
 def _read_task_file(task: HarborTask, file_id: str) -> str | None:
-    """Fetch the content for a file_id; None when the file isn't present."""
+    """Fetch the content for a file_id; None when the file isn't present.
+
+    task.toml and instruction.md have special handling (task.toml uses the
+    pre-captured raw text; instruction.md falls back to the inline `task.instruction`
+    field from task.toml when no instruction.md file is on disk). Everything else
+    is a direct lookup against the dict populated by walking the task dir.
+    """
     if file_id == "task.toml":
         return task.task_toml_raw or None
     if file_id == "instruction.md":
-        return task.instruction_md or task.instruction_inline
-    if file_id == "solution/patch.diff":
-        return task.oracle_patch
-    if file_id == "solution/solve.sh":
-        return task.solve_sh
-    if file_id == "tests/test.sh":
-        return task.test_sh
-    if file_id == "environment/Dockerfile":
-        return task.dockerfile
-    return None
+        return task.files.get("instruction.md") or task.instruction_inline
+    return task.files.get(file_id)
 
 
 def _build_file_tree(task: HarborTask) -> tuple[list[tuple[str, str]], dict[str, str]]:
     """Render a file-explorer-style choice list for a task.
 
+    Walks every file discovered under the task dir (via `task.files`) and groups
+    them by their first path segment. Order: Overview → top-level files
+    (task.toml, instruction.md, anything else) → folders alphabetically with
+    their children alphabetically. No hardcoded allowlist — what's on disk is
+    what gets shown.
+
     Returns:
       choices: list of (label, value) tuples for `gr.Radio`. Labels use unicode
                tree glyphs (📂, ├─, └─) so the radio reads like a file tree.
       folder_redirects: maps each folder pseudo-id to its first present child
                         file_id so clicking a folder header opens its first file.
-
-    Order is fixed: Overview first, then top-level files, then folders with
-    their children. Folders only appear when they have ≥1 present child.
     """
     choices: list[tuple[str, str]] = [("ⓘ  Overview", _OVERVIEW)]
     redirects: dict[str, str] = {}
 
-    # Top-level files (always under root)
+    # Bucket every discovered file by top-level dir ("" = at task root)
+    top_level: list[str] = []
+    by_folder: dict[str, list[str]] = {}
+    for path in sorted(task.files):
+        if "/" in path:
+            folder = path.split("/", 1)[0]
+            by_folder.setdefault(folder, []).append(path)
+        else:
+            top_level.append(path)
+
+    # Top-level files: task.toml first, then instruction.md (with inline
+    # fallback), then anything else alphabetically. Order is presentational.
     if (task.task_toml_raw or "").strip():
         choices.append(("📄  task.toml", "task.toml"))
-    if (task.instruction_md or task.instruction_inline):
+    if (task.files.get("instruction.md") or task.instruction_inline):
         choices.append(("📄  instruction.md", "instruction.md"))
-
-    def _maybe_folder(folder: str, children: list[tuple[str, str]]) -> None:
-        """Append `📂 folder/` + indented children, only if children non-empty.
-
-        `children` is a list of (basename, full_file_id) tuples.
-        """
-        present = [(b, fid) for (b, fid) in children if _read_task_file(task, fid) is not None]
-        if not present:
-            return
+    for path in sorted(top_level):
+        if path in ("task.toml", "instruction.md"):
+            continue   # already added
+        choices.append((f"📄  {path}", path))
+
+    # Folders alphabetically (environment / solution / tests / ...) with
+    # children alphabetical within each.
+    for folder in sorted(by_folder):
+        children = sorted(by_folder[folder])
+        if not children:
+            continue
         folder_id = f"{_FOLDER_PREFIX}{folder}"
         choices.append((f"📂  {folder}/", folder_id))
-        redirects[folder_id] = present[0][1]  # folder header → first child
-        for i, (basename, full_id) in enumerate(present):
-            glyph = "└─" if i == len(present) - 1 else "├─"
+        redirects[folder_id] = children[0]  # folder header → first child
+        for i, full_id in enumerate(children):
+            # Show the path *inside* the folder (handles nested subdirs too)
+            basename = full_id[len(folder) + 1:]   # strip "<folder>/"
+            glyph = "└─" if i == len(children) - 1 else "├─"
             choices.append((f"     {glyph}  {basename}", full_id))
 
-    _maybe_folder("solution", [
-        ("patch.diff", "solution/patch.diff"),
-        ("solve.sh", "solution/solve.sh"),
-    ])
-    _maybe_folder("tests", [
-        ("test.sh", "tests/test.sh"),
-    ])
-    _maybe_folder("environment", [
-        ("Dockerfile", "environment/Dockerfile"),
-    ])
-
     return choices, redirects
 
 

viewer/parse.py

@@ -61,6 +61,14 @@ class HarborTask:
     dockerfile: str | None = None
     task_toml_raw: str = ""
 
+    # Generic discovery: every readable text file under the task dir,
+    # keyed by path relative to the task root. e.g. "tests/grader.py",
+    # "environment/pull_bucket.py", "solution/patch.diff". Populated by
+    # load_task() walking the directory tree — the file viewer surfaces
+    # everything in here so the dataset is shown faithfully, not via
+    # a hardcoded allowlist.
+    files: dict[str, str] = field(default_factory=dict)
+
 
 def _read_text(path: Path) -> str | None:
     """Read a text file, return None if it doesn't exist."""
@@ -73,6 +81,42 @@ def _read_text(path: Path) -> str | None:
         return None
 
 
+# Files we never surface in the file viewer (binaries, caches, secrets).
+_SKIP_DIRS: set[str] = {".git", "__pycache__", ".cache", "node_modules", ".venv", ".pytest_cache", ".mypy_cache"}
+_SKIP_NAME_PREFIXES: tuple[str, ...] = (".DS_Store",)
+# Hard cap on file size — anything bigger we treat as non-displayable.
+_MAX_FILE_BYTES = 512 * 1024  # 512 KiB; viewer's code panel chokes well below this
+
+
+def _discover_task_files(task_dir: Path) -> dict[str, str]:
+    """Walk `task_dir` recursively and return every readable text file as
+    {relative_path: content}. Skips binaries, hidden noise, and oversized files.
+    Paths use forward slashes (POSIX-style) for stable file_ids across OSes.
+    """
+    out: dict[str, str] = {}
+    for path in sorted(task_dir.rglob("*")):
+        if not path.is_file():
+            continue
+        # Skip files inside excluded directories (any level)
+        if any(part in _SKIP_DIRS for part in path.relative_to(task_dir).parts):
+            continue
+        if path.name.startswith(_SKIP_NAME_PREFIXES):
+            continue
+        try:
+            size = path.stat().st_size
+        except OSError:
+            continue
+        if size > _MAX_FILE_BYTES:
+            continue
+        try:
+            content = path.read_text(encoding="utf-8")
+        except (UnicodeDecodeError, OSError):
+            continue
+        rel = path.relative_to(task_dir).as_posix()
+        out[rel] = content
+    return out
+
+
 def _discover_task_roots(dataset_root: Path) -> list[Path]:
     """Find every directory under `dataset_root` that contains a `task.toml`.
 
@@ -136,6 +180,10 @@ def load_task(dataset_root: Path, task_id: str) -> HarborTask:
     if repo2env is not None and not isinstance(repo2env, dict):
         repo2env = None
 
+    # Walk the whole task dir so any present file (grader.py, pull_bucket.py,
+    # helper modules, multiple test scripts, ...) ends up in the viewer.
+    files = _discover_task_files(task_dir)
+
     return HarborTask(
         id=task_id,
         root=task_dir,
@@ -150,10 +198,13 @@ def load_task(dataset_root: Path, task_id: str) -> HarborTask:
         agent_timeout_sec=agent_block.get("timeout_sec"),
         verifier_timeout_sec=verifier_block.get("timeout_sec"),
         repo2env=repo2env,
-        instruction_md=_read_text(task_dir / "instruction.md"),
-        oracle_patch=_read_text(task_dir / "solution" / "patch.diff"),
-        solve_sh=_read_text(task_dir / "solution" / "solve.sh"),
-        test_sh=_read_text(task_dir / "tests" / "test.sh"),
-        dockerfile=_read_text(task_dir / "environment" / "Dockerfile"),
+        # Convenience fields — kept populated for backwards compat with callers
+        # that reach in by name. Source of truth for the viewer is `files`.
+        instruction_md=files.get("instruction.md"),
+        oracle_patch=files.get("solution/patch.diff"),
+        solve_sh=files.get("solution/solve.sh"),
+        test_sh=files.get("tests/test.sh"),
+        dockerfile=files.get("environment/Dockerfile"),
         task_toml_raw=raw,
+        files=files,
     )