Spaces:
Running on Zero
Harden GAIA agent: provider fallback, rate-limit pacing, context trimming
Browse filesScored 45% (9/20) on the official scoring API, up from a baseline that could
not complete a run at all. All changes stay within the existing single-file
CodeAgent architecture; no framework or tooling was swapped.
Model access
- Pin the model id explicitly instead of relying on the smolagents default,
which changes between releases and made Space rebuilds non-reproducible.
- Support any OpenAI-compatible endpoint via GAIA_BASE_URL/GAIA_API_KEY, since
HF's free monthly credits return 402 on every call once depleted, failing all
20 questions at once.
- GAIA_MODEL_ID accepts a comma-separated chain; exhausting one model's daily
token quota switches to the next rather than ending the run.
Rate limits
- Pace requests below the provider ceiling (GAIA_RPM) instead of discovering it
through 429s.
- Honour the provider's own retry hint; a blind doubling backoff was routinely
shorter than the interval the server asked for.
- Disable smolagents' internal retryer. Its 3 attempts at 60s base, doubling and
jittered, ran inside our own attempts, so one throttled call could sleep for
over half an hour.
- Fail fast on daily-quota and 413 errors. Both arrive dressed as rate limits
but no backoff clears them, and retrying burned the wall-clock left for the
remaining questions.
- Retry the 400 "Tool choice is none, but model called a tool" rejection; it is
a sampling artifact that clears on resample.
Context growth
- Trim older steps' observations and model output via a step callback. Every
past step is replayed into every later call, so trajectories grew until a
single request exceeded the per-minute token limit outright.
- Give web_search and visit_webpage smaller output budgets than the library
defaults for the same reason.
Correctness and robustness
- Send a descriptive User-Agent from visit_webpage; the stock tool sends none
and Wikipedia answers 403.
- Tell the agent explicitly when an attachment could not be retrieved, so it
stops inventing file contents. All five attachments 404 server-side.
- Retry transient web_search failures rather than burning an agent step.
- Raise max_steps and authorise a few more stdlib imports for multi-hop
research questions.
- Add a dev-only single-question runner to the UI; it submits nothing.
- Fix a startup log line that double-appended .hf.space to SPACE_HOST.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
- .gitignore +4 -0
- app.py +395 -28
- requirements.txt +3 -2
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
.venv/
|
| 2 |
+
__pycache__/
|
| 3 |
+
*.pyc
|
| 4 |
+
.gaia_env
|
|
@@ -1,7 +1,9 @@
|
|
| 1 |
import spaces # must be the first import — required by ZeroGPU Spaces
|
| 2 |
|
| 3 |
import os
|
|
|
|
| 4 |
import tempfile
|
|
|
|
| 5 |
from pathlib import Path
|
| 6 |
from typing import Optional
|
| 7 |
|
|
@@ -9,11 +11,17 @@ import gradio as gr
|
|
| 9 |
import requests
|
| 10 |
import pandas as pd
|
| 11 |
|
| 12 |
-
from smolagents import CodeAgent, InferenceClientModel
|
|
|
|
| 13 |
|
| 14 |
# --- Constants ---
|
| 15 |
DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
|
| 16 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 17 |
# This Space's free-tier hardware is ZeroGPU, which refuses to start unless
|
| 18 |
# at least one function is decorated with @spaces.GPU. The agent itself never
|
| 19 |
# touches a GPU (it only calls HF Inference Providers over HTTP), so this
|
|
@@ -22,6 +30,12 @@ DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
|
|
| 22 |
def _zerogpu_startup_check():
|
| 23 |
return None
|
| 24 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 25 |
# Instructions appended to every question so the CodeAgent behaves like a
|
| 26 |
# GAIA-solving agent rather than a free-form chatbot. Kept as plain task
|
| 27 |
# framing (not a tool) to avoid adding moving parts.
|
|
@@ -34,22 +48,223 @@ Rules:
|
|
| 34 |
sorting, filtering, or any data processing rather than doing it in your head.
|
| 35 |
4. If an attachment path is given below, open and inspect it with Python
|
| 36 |
before answering — never guess its contents.
|
| 37 |
-
5.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 38 |
before finalizing.
|
| 39 |
-
|
| 40 |
- a bare number (no commas, no units unless the question asks for units)
|
| 41 |
- a bare string (no "The answer is", no trailing period, no quotes)
|
| 42 |
-
- a comma-separated list only if asked for a list,
|
| 43 |
-
|
|
|
|
| 44 |
Do not include your reasoning or the words "FINAL ANSWER" in that value.
|
| 45 |
"""
|
| 46 |
|
| 47 |
-
MAX_STEPS =
|
| 48 |
ADDITIONAL_IMPORTS = [
|
| 49 |
"pandas", "numpy", "csv", "json", "re", "math", "statistics",
|
| 50 |
"datetime", "itertools", "collections", "io", "os", "pypdf", "openpyxl",
|
|
|
|
| 51 |
]
|
| 52 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 53 |
|
| 54 |
class GAIAAgent:
|
| 55 |
"""
|
|
@@ -62,35 +277,103 @@ class GAIAAgent:
|
|
| 62 |
hf_token = os.environ.get("HF_TOKEN")
|
| 63 |
model_id = os.environ.get("GAIA_MODEL_ID") # optional override
|
| 64 |
provider = os.environ.get("GAIA_PROVIDER") # optional override; unset = library default ("auto")
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 75 |
)
|
| 76 |
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
|
|
|
| 83 |
|
| 84 |
self.agent = CodeAgent(
|
| 85 |
tools=[],
|
| 86 |
model=model,
|
| 87 |
-
add_base_tools=True, # gives
|
| 88 |
additional_authorized_imports=ADDITIONAL_IMPORTS,
|
| 89 |
max_steps=MAX_STEPS,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 90 |
)
|
| 91 |
print("GAIAAgent initialized (smolagents CodeAgent).")
|
| 92 |
|
| 93 |
-
def __call__(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 94 |
task = question + "\n\n" + TASK_RULES
|
| 95 |
if file_path:
|
| 96 |
task += (
|
|
@@ -98,6 +381,16 @@ class GAIAAgent:
|
|
| 98 |
f"path: {file_path}\nOpen it with Python and inspect its "
|
| 99 |
f"contents before answering.\n"
|
| 100 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 101 |
raw_answer = self.agent.run(task)
|
| 102 |
return clean_final_answer(raw_answer)
|
| 103 |
|
|
@@ -122,17 +415,75 @@ def download_task_file(api_url: str, task_id: str, file_name: str) -> Optional[s
|
|
| 122 |
return None
|
| 123 |
try:
|
| 124 |
resp = requests.get(f"{api_url}/files/{task_id}", timeout=30)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 125 |
resp.raise_for_status()
|
| 126 |
out_dir = Path(tempfile.gettempdir()) / "gaia_files"
|
| 127 |
out_dir.mkdir(exist_ok=True)
|
| 128 |
out_path = out_dir / file_name
|
| 129 |
out_path.write_bytes(resp.content)
|
|
|
|
| 130 |
return str(out_path)
|
| 131 |
except Exception as e:
|
| 132 |
-
print(f"Could not download file for task {task_id}: {e}")
|
| 133 |
return None
|
| 134 |
|
| 135 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 136 |
def run_and_submit_all(profile: gr.OAuthProfile | None):
|
| 137 |
"""
|
| 138 |
Fetches all questions, runs the GAIAAgent on them (downloading any
|
|
@@ -196,11 +547,11 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
|
|
| 196 |
file_path = download_task_file(api_url, task_id, file_name)
|
| 197 |
|
| 198 |
try:
|
| 199 |
-
submitted_answer = agent(question_text, file_path=file_path)
|
| 200 |
answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
|
| 201 |
results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})
|
| 202 |
except Exception as e:
|
| 203 |
-
print(f"Error running agent on task {task_id}: {e}")
|
| 204 |
results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": f"AGENT ERROR: {e}"})
|
| 205 |
|
| 206 |
if not answers_payload:
|
|
@@ -283,6 +634,22 @@ with gr.Blocks() as demo:
|
|
| 283 |
outputs=[status_output, results_table]
|
| 284 |
)
|
| 285 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 286 |
if __name__ == "__main__":
|
| 287 |
print("\n" + "-" * 30 + " App Starting " + "-" * 30)
|
| 288 |
space_host_startup = os.getenv("SPACE_HOST")
|
|
@@ -290,7 +657,7 @@ if __name__ == "__main__":
|
|
| 290 |
|
| 291 |
if space_host_startup:
|
| 292 |
print(f"✅ SPACE_HOST found: {space_host_startup}")
|
| 293 |
-
print(f" Runtime URL
|
| 294 |
else:
|
| 295 |
print("ℹ️ SPACE_HOST environment variable not found (running locally?).")
|
| 296 |
|
|
|
|
| 1 |
import spaces # must be the first import — required by ZeroGPU Spaces
|
| 2 |
|
| 3 |
import os
|
| 4 |
+
import re
|
| 5 |
import tempfile
|
| 6 |
+
import time
|
| 7 |
from pathlib import Path
|
| 8 |
from typing import Optional
|
| 9 |
|
|
|
|
| 11 |
import requests
|
| 12 |
import pandas as pd
|
| 13 |
|
| 14 |
+
from smolagents import CodeAgent, InferenceClientModel, OpenAIServerModel
|
| 15 |
+
from smolagents.default_tools import DuckDuckGoSearchTool, VisitWebpageTool
|
| 16 |
|
| 17 |
# --- Constants ---
|
| 18 |
DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
|
| 19 |
|
| 20 |
+
# Pinned explicitly rather than relying on InferenceClientModel's own default,
|
| 21 |
+
# which changes between smolagents releases. The GAIA_MODEL_ID env var still
|
| 22 |
+
# overrides this if a different model is ever needed.
|
| 23 |
+
DEFAULT_MODEL_ID = "meta-llama/Llama-3.3-70B-Instruct"
|
| 24 |
+
|
| 25 |
# This Space's free-tier hardware is ZeroGPU, which refuses to start unless
|
| 26 |
# at least one function is decorated with @spaces.GPU. The agent itself never
|
| 27 |
# touches a GPU (it only calls HF Inference Providers over HTTP), so this
|
|
|
|
| 30 |
def _zerogpu_startup_check():
|
| 31 |
return None
|
| 32 |
|
| 33 |
+
# Free provider tiers cap requests per minute (Google's Gemini free tier
|
| 34 |
+
# reports "limit: 20" for gemini-2.5-flash). Pacing our own calls below that
|
| 35 |
+
# ceiling is far cheaper than discovering it via 429s and backoff. Override
|
| 36 |
+
# with GAIA_RPM when moving to a tier with a different budget.
|
| 37 |
+
DEFAULT_REQUESTS_PER_MINUTE = 15.0
|
| 38 |
+
|
| 39 |
# Instructions appended to every question so the CodeAgent behaves like a
|
| 40 |
# GAIA-solving agent rather than a free-form chatbot. Kept as plain task
|
| 41 |
# framing (not a tool) to avoid adding moving parts.
|
|
|
|
| 48 |
sorting, filtering, or any data processing rather than doing it in your head.
|
| 49 |
4. If an attachment path is given below, open and inspect it with Python
|
| 50 |
before answering — never guess its contents.
|
| 51 |
+
5. If web_search fails or comes back empty, do not give up on the step: retry
|
| 52 |
+
with a differently worded query, or call visit_webpage directly on a likely
|
| 53 |
+
source URL (for example
|
| 54 |
+
https://en.wikipedia.org/w/index.php?search=YOUR+QUERY, or the article URL
|
| 55 |
+
itself). Do not try to import external packages such as wikipedia,
|
| 56 |
+
requests-html, bs4 or googlesearch — they are not installed, and attempting
|
| 57 |
+
them only wastes a step.
|
| 58 |
+
6. Where practical, verify an important intermediate result a second way
|
| 59 |
before finalizing.
|
| 60 |
+
7. Call final_answer(...) with ONLY the exact value requested:
|
| 61 |
- a bare number (no commas, no units unless the question asks for units)
|
| 62 |
- a bare string (no "The answer is", no trailing period, no quotes)
|
| 63 |
+
- a comma-separated list only if asked for a list, in exactly the order the
|
| 64 |
+
question asks for; separate elements with a comma and a single space
|
| 65 |
+
unless the question specifies a different separator
|
| 66 |
Do not include your reasoning or the words "FINAL ANSWER" in that value.
|
| 67 |
"""
|
| 68 |
|
| 69 |
+
MAX_STEPS = 12
|
| 70 |
ADDITIONAL_IMPORTS = [
|
| 71 |
"pandas", "numpy", "csv", "json", "re", "math", "statistics",
|
| 72 |
"datetime", "itertools", "collections", "io", "os", "pypdf", "openpyxl",
|
| 73 |
+
"requests", "unicodedata", "time",
|
| 74 |
]
|
| 75 |
|
| 76 |
+
# Every past step's observations are replayed into every later model call, so a
|
| 77 |
+
# trajectory grows quadratically and eventually exceeds what a free tier will
|
| 78 |
+
# accept in one request — Groq returns HTTP 413 "Request too large ... Limit
|
| 79 |
+
# 12000, Requested 12396" and no amount of waiting fixes it, because a single
|
| 80 |
+
# call already exceeds the whole per-minute budget. Recent observations are what
|
| 81 |
+
# the agent is reasoning about; older ones only need to stay recognizable.
|
| 82 |
+
RECENT_STEPS_KEPT_FULL = 2
|
| 83 |
+
RECENT_OBSERVATION_CHARS = 8000
|
| 84 |
+
OLDER_OBSERVATION_CHARS = 1500
|
| 85 |
+
# The model's own output is replayed as an assistant message too, and reasoning
|
| 86 |
+
# models are verbose enough that untrimmed history alone can breach the limit.
|
| 87 |
+
# The code it wrote is the part worth keeping, so older steps keep only a head.
|
| 88 |
+
RECENT_OUTPUT_CHARS = 4000
|
| 89 |
+
OLDER_OUTPUT_CHARS = 1000
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def trim_observations(memory_step, agent=None) -> None:
|
| 93 |
+
"""Step callback: shrink observations of all but the newest few steps.
|
| 94 |
+
|
| 95 |
+
Registered on ActionStep, so it runs after each action and keeps the next
|
| 96 |
+
request inside the provider's per-request ceiling.
|
| 97 |
+
"""
|
| 98 |
+
def _clip(text, budget):
|
| 99 |
+
if not isinstance(text, str) or len(text) <= budget:
|
| 100 |
+
return text
|
| 101 |
+
return (
|
| 102 |
+
text[:budget]
|
| 103 |
+
+ f"\n...[{len(text) - budget} characters truncated to stay within "
|
| 104 |
+
f"the model's request size limit]"
|
| 105 |
+
)
|
| 106 |
+
|
| 107 |
+
steps = [s for s in getattr(agent, "memory", None).steps if hasattr(s, "observations")]
|
| 108 |
+
for index, step in enumerate(steps):
|
| 109 |
+
is_recent = index >= len(steps) - RECENT_STEPS_KEPT_FULL
|
| 110 |
+
if step.observations:
|
| 111 |
+
step.observations = _clip(
|
| 112 |
+
step.observations,
|
| 113 |
+
RECENT_OBSERVATION_CHARS if is_recent else OLDER_OBSERVATION_CHARS,
|
| 114 |
+
)
|
| 115 |
+
if step.model_output:
|
| 116 |
+
step.model_output = _clip(
|
| 117 |
+
step.model_output,
|
| 118 |
+
RECENT_OUTPUT_CHARS if is_recent else OLDER_OUTPUT_CHARS,
|
| 119 |
+
)
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
class RetryingWebSearchTool(DuckDuckGoSearchTool):
|
| 123 |
+
"""Same tool the base toolkit provides, but a rate-limited or flaky
|
| 124 |
+
DuckDuckGo response is retried instead of burning one of the agent's
|
| 125 |
+
steps. Name/description/signature are inherited unchanged, so the agent's
|
| 126 |
+
system prompt is identical to the stock tool's."""
|
| 127 |
+
|
| 128 |
+
max_attempts = 3
|
| 129 |
+
backoff_seconds = 3.0
|
| 130 |
+
|
| 131 |
+
def forward(self, query: str) -> str:
|
| 132 |
+
last_error = None
|
| 133 |
+
for attempt in range(self.max_attempts):
|
| 134 |
+
try:
|
| 135 |
+
return super().forward(query)
|
| 136 |
+
except Exception as e:
|
| 137 |
+
last_error = e
|
| 138 |
+
print(f"web_search attempt {attempt + 1}/{self.max_attempts} failed: {e}")
|
| 139 |
+
if attempt < self.max_attempts - 1:
|
| 140 |
+
time.sleep(self.backoff_seconds * (attempt + 1))
|
| 141 |
+
raise RuntimeError(
|
| 142 |
+
f"web_search failed after {self.max_attempts} attempts: {last_error}. "
|
| 143 |
+
"Try visit_webpage on a likely source URL instead."
|
| 144 |
+
)
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
class _RetryOnThrottleMixin:
|
| 148 |
+
"""Retries transient rate-limit / server errors from the inference endpoint.
|
| 149 |
+
Free provider tiers throttle aggressively, and without this a single 429
|
| 150 |
+
aborts the whole question. Non-transient errors (401, 402 out of credits,
|
| 151 |
+
400 bad request) are re-raised immediately — retrying those is pointless.
|
| 152 |
+
|
| 153 |
+
Both model classes below are constructed with retry=False, which disables
|
| 154 |
+
smolagents' own retryer. Leaving it on nests two exponential backoffs: its
|
| 155 |
+
3 attempts (60s base, doubling, jittered) run inside each of our attempts,
|
| 156 |
+
so one throttled call can sleep for over half an hour. This layer replaces
|
| 157 |
+
it because it can read the provider's own retry hint instead of guessing."""
|
| 158 |
+
|
| 159 |
+
max_attempts = 6
|
| 160 |
+
backoff_seconds = 20.0
|
| 161 |
+
RETRYABLE = (
|
| 162 |
+
"429", "500", "502", "503", "504", "rate limit", "too many requests", "timeout",
|
| 163 |
+
# Some models occasionally emit a tool call where CodeAgent expects a
|
| 164 |
+
# code block, and the provider rejects the request outright with
|
| 165 |
+
# 400 "Tool choice is none, but model called a tool". It is a sampling
|
| 166 |
+
# artifact rather than a bad prompt, so re-asking usually succeeds.
|
| 167 |
+
"tool_use_failed",
|
| 168 |
+
)
|
| 169 |
+
# A per-day quota and an over-sized single request both arrive dressed as
|
| 170 |
+
# rate limits, but neither clears within any backoff we would sit through:
|
| 171 |
+
# the daily bucket refills hours later, and a request that alone exceeds the
|
| 172 |
+
# per-minute ceiling will be exactly as large on the next attempt. Failing
|
| 173 |
+
# this question immediately leaves budget and wall-clock for the rest.
|
| 174 |
+
FATAL = ("tokens per day", "tpd", "request too large", "413")
|
| 175 |
+
# Providers usually say how long to wait; obeying that beats guessing.
|
| 176 |
+
# Gemini phrases it "Please retry in 32.290648364s", OpenAI-style APIs
|
| 177 |
+
# "retry after 12 seconds".
|
| 178 |
+
_RETRY_HINT = re.compile(r"retry(?:\s+after)?\s+in\s+([0-9.]+)\s*s|retry after ([0-9.]+)")
|
| 179 |
+
|
| 180 |
+
# Filled in by GAIAAgent when GAIA_MODEL_ID lists more than one model.
|
| 181 |
+
fallback_model_ids: list = []
|
| 182 |
+
|
| 183 |
+
def generate(self, *args, **kwargs):
|
| 184 |
+
last_error = None
|
| 185 |
+
for attempt in range(self.max_attempts):
|
| 186 |
+
try:
|
| 187 |
+
return super().generate(*args, **kwargs)
|
| 188 |
+
except Exception as e:
|
| 189 |
+
message = str(e).lower()
|
| 190 |
+
if any(token in message for token in self.FATAL):
|
| 191 |
+
# Daily quotas are per-model, so a sibling model on the same
|
| 192 |
+
# account usually still has budget. Switching costs one call
|
| 193 |
+
# and rescues every remaining question; without it the run
|
| 194 |
+
# ends here no matter how much wall-clock is left.
|
| 195 |
+
if "tokens per day" in message and self.fallback_model_ids:
|
| 196 |
+
self.model_id = self.fallback_model_ids.pop(0)
|
| 197 |
+
print(
|
| 198 |
+
f"Daily token quota exhausted; switching to "
|
| 199 |
+
f"fallback model {self.model_id}"
|
| 200 |
+
)
|
| 201 |
+
continue
|
| 202 |
+
raise
|
| 203 |
+
if not any(token in message for token in self.RETRYABLE):
|
| 204 |
+
raise
|
| 205 |
+
last_error = e
|
| 206 |
+
# A malformed generation clears on the next sample, so re-ask
|
| 207 |
+
# straight away rather than serving a rate-limit-sized backoff.
|
| 208 |
+
if "tool_use_failed" in message:
|
| 209 |
+
wait = 1.0
|
| 210 |
+
else:
|
| 211 |
+
wait = self.backoff_seconds * (attempt + 1)
|
| 212 |
+
hint = self._RETRY_HINT.search(message)
|
| 213 |
+
if hint:
|
| 214 |
+
# +2s of slack so we come back after the window, not on its edge
|
| 215 |
+
wait = max(wait, float(hint.group(1) or hint.group(2)) + 2.0)
|
| 216 |
+
print(
|
| 217 |
+
f"Model call attempt {attempt + 1}/{self.max_attempts} failed "
|
| 218 |
+
f"({type(e).__name__}: {str(e)[:200]}); retrying in {wait:.0f}s"
|
| 219 |
+
)
|
| 220 |
+
if attempt < self.max_attempts - 1:
|
| 221 |
+
time.sleep(wait)
|
| 222 |
+
raise last_error
|
| 223 |
+
|
| 224 |
+
|
| 225 |
+
class RetryingInferenceClientModel(_RetryOnThrottleMixin, InferenceClientModel):
|
| 226 |
+
"""HF Inference Providers, with throttle retries."""
|
| 227 |
+
|
| 228 |
+
|
| 229 |
+
class RetryingOpenAIServerModel(_RetryOnThrottleMixin, OpenAIServerModel):
|
| 230 |
+
"""Any OpenAI-compatible endpoint, with throttle retries. InferenceClient
|
| 231 |
+
itself cannot take a model name and a base_url together, so custom
|
| 232 |
+
endpoints go through this class instead."""
|
| 233 |
+
|
| 234 |
+
|
| 235 |
+
class IdentifiedVisitWebpageTool(VisitWebpageTool):
|
| 236 |
+
"""The stock tool calls requests.get() with no User-Agent, so Wikipedia and
|
| 237 |
+
several other sources answer 403 Forbidden — verified against
|
| 238 |
+
en.wikipedia.org. Sending a descriptive User-Agent (as Wikimedia's bot
|
| 239 |
+
policy asks for) is the whole fix; everything else is inherited."""
|
| 240 |
+
|
| 241 |
+
USER_AGENT = (
|
| 242 |
+
"GAIA-Agent/1.0 (HF Agents Course Unit 4 final assignment; "
|
| 243 |
+
"+https://huggingface.co/spaces/sumit1703/Final_Assignment_Sumit)"
|
| 244 |
+
)
|
| 245 |
+
|
| 246 |
+
def forward(self, url: str) -> str:
|
| 247 |
+
import re
|
| 248 |
+
|
| 249 |
+
import requests as _requests
|
| 250 |
+
from markdownify import markdownify
|
| 251 |
+
from requests.exceptions import RequestException
|
| 252 |
+
|
| 253 |
+
try:
|
| 254 |
+
response = _requests.get(
|
| 255 |
+
url, timeout=20, headers={"User-Agent": self.USER_AGENT}
|
| 256 |
+
)
|
| 257 |
+
response.raise_for_status()
|
| 258 |
+
markdown_content = markdownify(response.text).strip()
|
| 259 |
+
markdown_content = re.sub(r"\n{3,}", "\n\n", markdown_content)
|
| 260 |
+
return self._truncate_content(markdown_content, self.max_output_length)
|
| 261 |
+
except _requests.exceptions.Timeout:
|
| 262 |
+
return "The request timed out. Please try again later or check the URL."
|
| 263 |
+
except RequestException as e:
|
| 264 |
+
return f"Error fetching the webpage: {str(e)}"
|
| 265 |
+
except Exception as e:
|
| 266 |
+
return f"An unexpected error occurred: {str(e)}"
|
| 267 |
+
|
| 268 |
|
| 269 |
class GAIAAgent:
|
| 270 |
"""
|
|
|
|
| 277 |
hf_token = os.environ.get("HF_TOKEN")
|
| 278 |
model_id = os.environ.get("GAIA_MODEL_ID") # optional override
|
| 279 |
provider = os.environ.get("GAIA_PROVIDER") # optional override; unset = library default ("auto")
|
| 280 |
+
base_url = os.environ.get("GAIA_BASE_URL") # optional OpenAI-compatible endpoint
|
| 281 |
+
api_key = os.environ.get("GAIA_API_KEY") # key for that endpoint
|
| 282 |
+
rpm = float(os.environ.get("GAIA_RPM") or DEFAULT_REQUESTS_PER_MINUTE)
|
| 283 |
+
|
| 284 |
+
# GAIA_MODEL_ID may list several models, best first. Later entries are
|
| 285 |
+
# used only when an earlier one exhausts its daily token quota.
|
| 286 |
+
model_ids = [m.strip() for m in (model_id or "").split(",") if m.strip()]
|
| 287 |
+
model_id = model_ids[0] if model_ids else None
|
| 288 |
+
fallback_model_ids = model_ids[1:]
|
| 289 |
+
|
| 290 |
+
# Two ways to reach a model, both through the same InferenceClientModel:
|
| 291 |
+
# 1. (default) HF Inference Providers, billed to HF_TOKEN's account.
|
| 292 |
+
# 2. any OpenAI-compatible endpoint, when GAIA_BASE_URL is set. This
|
| 293 |
+
# exists because HF's free monthly credits are small and a
|
| 294 |
+
# depleted account returns 402 on every single call, which fails
|
| 295 |
+
# all 20 questions at once.
|
| 296 |
+
# Fail fast and loud either way, instead of letting all 20 questions die
|
| 297 |
+
# silently at Step 1 with a generic error from deep inside smolagents.
|
| 298 |
+
if base_url:
|
| 299 |
+
print(f"Using custom OpenAI-compatible endpoint: {base_url}")
|
| 300 |
+
print(f"GAIA_API_KEY present: {bool(api_key)}")
|
| 301 |
+
if not api_key:
|
| 302 |
+
raise RuntimeError(
|
| 303 |
+
"GAIA_BASE_URL is set but GAIA_API_KEY is not. Add a secret "
|
| 304 |
+
"named exactly GAIA_API_KEY holding the key for that endpoint."
|
| 305 |
+
)
|
| 306 |
+
if not model_id:
|
| 307 |
+
raise RuntimeError(
|
| 308 |
+
"GAIA_BASE_URL is set but GAIA_MODEL_ID is not. A custom "
|
| 309 |
+
"endpoint needs its own model name (for example "
|
| 310 |
+
"'llama-3.3-70b-versatile' on Groq), since provider model "
|
| 311 |
+
"ids differ from Hugging Face repo ids."
|
| 312 |
+
)
|
| 313 |
+
# Bound each HTTP call: the openai SDK otherwise waits up to 10
|
| 314 |
+
# minutes and silently retries, so one throttled call can stall a
|
| 315 |
+
# whole question. Our own retry loop handles the backoff instead.
|
| 316 |
+
model = RetryingOpenAIServerModel(
|
| 317 |
+
model_id=model_id,
|
| 318 |
+
api_base=base_url,
|
| 319 |
+
api_key=api_key,
|
| 320 |
+
requests_per_minute=rpm,
|
| 321 |
+
retry=False,
|
| 322 |
+
client_kwargs={"timeout": 90, "max_retries": 0},
|
| 323 |
+
)
|
| 324 |
+
model_kwargs = {"model_id": model_id}
|
| 325 |
+
else:
|
| 326 |
+
print(f"HF_TOKEN present: {bool(hf_token)}")
|
| 327 |
+
if not hf_token:
|
| 328 |
+
raise RuntimeError(
|
| 329 |
+
"HF_TOKEN is not set in this process. Go to Space Settings > "
|
| 330 |
+
"Variables and secrets and confirm a secret named exactly "
|
| 331 |
+
"HF_TOKEN exists, then restart the Space (adding a secret does "
|
| 332 |
+
"not always hot-reload a running container)."
|
| 333 |
+
)
|
| 334 |
+
model_kwargs = {"token": hf_token, "model_id": model_id or DEFAULT_MODEL_ID}
|
| 335 |
+
if provider:
|
| 336 |
+
model_kwargs["provider"] = provider
|
| 337 |
+
model = RetryingInferenceClientModel(
|
| 338 |
+
requests_per_minute=rpm, retry=False, **model_kwargs
|
| 339 |
)
|
| 340 |
|
| 341 |
+
# Instance attribute, so exhausting one model's quota never mutates the
|
| 342 |
+
# class-level default shared by every other agent in the process.
|
| 343 |
+
model.fallback_model_ids = list(fallback_model_ids)
|
| 344 |
+
|
| 345 |
+
print(f"Model: {model_kwargs['model_id']} (paced at {rpm:g} requests/minute)")
|
| 346 |
+
if fallback_model_ids:
|
| 347 |
+
print(f"Fallback models on daily quota exhaustion: {', '.join(fallback_model_ids)}")
|
| 348 |
|
| 349 |
self.agent = CodeAgent(
|
| 350 |
tools=[],
|
| 351 |
model=model,
|
| 352 |
+
add_base_tools=True, # gives web_search + visit_webpage
|
| 353 |
additional_authorized_imports=ADDITIONAL_IMPORTS,
|
| 354 |
max_steps=MAX_STEPS,
|
| 355 |
+
step_callbacks=[trim_observations],
|
| 356 |
+
)
|
| 357 |
+
# add_base_tools installs the stock tools last, so swap in the hardened
|
| 358 |
+
# subclasses afterwards rather than passing them via tools=[].
|
| 359 |
+
# Both are given smaller output budgets than the library defaults
|
| 360 |
+
# (10 results, 40000 characters). Every tool result is replayed into the
|
| 361 |
+
# next model call, so a single stock visit_webpage is roughly 10k tokens
|
| 362 |
+
# — most of a free tier's whole per-minute allowance, spent on page
|
| 363 |
+
# boilerplate. Trimming keeps trajectories inside the budget and makes
|
| 364 |
+
# each step cheaper without losing the part of the page that matters.
|
| 365 |
+
self.agent.tools["web_search"] = RetryingWebSearchTool(max_results=6)
|
| 366 |
+
self.agent.tools["visit_webpage"] = IdentifiedVisitWebpageTool(
|
| 367 |
+
max_output_length=20000
|
| 368 |
)
|
| 369 |
print("GAIAAgent initialized (smolagents CodeAgent).")
|
| 370 |
|
| 371 |
+
def __call__(
|
| 372 |
+
self,
|
| 373 |
+
question: str,
|
| 374 |
+
file_path: Optional[str] = None,
|
| 375 |
+
file_name: Optional[str] = None,
|
| 376 |
+
) -> str:
|
| 377 |
task = question + "\n\n" + TASK_RULES
|
| 378 |
if file_path:
|
| 379 |
task += (
|
|
|
|
| 381 |
f"path: {file_path}\nOpen it with Python and inspect its "
|
| 382 |
f"contents before answering.\n"
|
| 383 |
)
|
| 384 |
+
elif file_name:
|
| 385 |
+
# The question ships an attachment but the scoring API could not
|
| 386 |
+
# serve it. Say so, otherwise the agent invents file contents.
|
| 387 |
+
task += (
|
| 388 |
+
f"\nNOTE: this task references an attachment ({file_name}) but "
|
| 389 |
+
f"it could not be retrieved from the evaluation server, so you "
|
| 390 |
+
f"do not have it. Do not pretend to open or read it. Answer "
|
| 391 |
+
f"from the question text and web research alone if that is "
|
| 392 |
+
f"possible; otherwise give your best supported answer.\n"
|
| 393 |
+
)
|
| 394 |
raw_answer = self.agent.run(task)
|
| 395 |
return clean_final_answer(raw_answer)
|
| 396 |
|
|
|
|
| 415 |
return None
|
| 416 |
try:
|
| 417 |
resp = requests.get(f"{api_url}/files/{task_id}", timeout=30)
|
| 418 |
+
if resp.status_code == 404:
|
| 419 |
+
# Distinguish "the evaluation server has no file mapped for this
|
| 420 |
+
# task" from a transport failure — they need different follow-ups.
|
| 421 |
+
print(
|
| 422 |
+
f"Attachment unavailable for task {task_id} ({file_name}): "
|
| 423 |
+
f"server returned 404 ({resp.text[:200]})"
|
| 424 |
+
)
|
| 425 |
+
return None
|
| 426 |
resp.raise_for_status()
|
| 427 |
out_dir = Path(tempfile.gettempdir()) / "gaia_files"
|
| 428 |
out_dir.mkdir(exist_ok=True)
|
| 429 |
out_path = out_dir / file_name
|
| 430 |
out_path.write_bytes(resp.content)
|
| 431 |
+
print(f"Downloaded attachment for task {task_id}: {out_path} ({len(resp.content)} bytes)")
|
| 432 |
return str(out_path)
|
| 433 |
except Exception as e:
|
| 434 |
+
print(f"Could not download file for task {task_id}: {type(e).__name__}: {e}")
|
| 435 |
return None
|
| 436 |
|
| 437 |
|
| 438 |
+
def run_single_question(task_id: str = ""):
|
| 439 |
+
"""Development helper: pull one task (a specific task_id, or a random one
|
| 440 |
+
from /random-question when left blank), run the agent on it, and show the
|
| 441 |
+
result. Submits nothing — this exists so individual tasks can be debugged
|
| 442 |
+
without spending a full 20-question submission."""
|
| 443 |
+
api_url = DEFAULT_API_URL
|
| 444 |
+
try:
|
| 445 |
+
task_id = (task_id or "").strip()
|
| 446 |
+
if task_id:
|
| 447 |
+
resp = requests.get(f"{api_url}/questions", timeout=15)
|
| 448 |
+
resp.raise_for_status()
|
| 449 |
+
matches = [q for q in resp.json() if q.get("task_id") == task_id]
|
| 450 |
+
if not matches:
|
| 451 |
+
return f"No question found with task_id {task_id}.", ""
|
| 452 |
+
item = matches[0]
|
| 453 |
+
else:
|
| 454 |
+
resp = requests.get(f"{api_url}/random-question", timeout=15)
|
| 455 |
+
resp.raise_for_status()
|
| 456 |
+
item = resp.json()
|
| 457 |
+
except Exception as e:
|
| 458 |
+
return f"Error fetching question: {type(e).__name__}: {e}", ""
|
| 459 |
+
|
| 460 |
+
task_id = item.get("task_id")
|
| 461 |
+
question_text = item.get("question")
|
| 462 |
+
file_name = item.get("file_name")
|
| 463 |
+
|
| 464 |
+
file_path = download_task_file(api_url, task_id, file_name) if file_name else None
|
| 465 |
+
if not file_name:
|
| 466 |
+
attachment_status = "none"
|
| 467 |
+
elif file_path:
|
| 468 |
+
attachment_status = f"{file_name} -> {file_path}"
|
| 469 |
+
else:
|
| 470 |
+
attachment_status = f"{file_name} (UNAVAILABLE - server returned no file)"
|
| 471 |
+
|
| 472 |
+
header = f"Task ID: {task_id}\nAttachment: {attachment_status}\n\nQuestion:\n{question_text}"
|
| 473 |
+
|
| 474 |
+
try:
|
| 475 |
+
agent = GAIAAgent()
|
| 476 |
+
except Exception as e:
|
| 477 |
+
return f"{header}\n\nError initializing agent: {e}", ""
|
| 478 |
+
|
| 479 |
+
try:
|
| 480 |
+
answer = agent(question_text, file_path=file_path, file_name=file_name)
|
| 481 |
+
return header, answer
|
| 482 |
+
except Exception as e:
|
| 483 |
+
print(f"Error running agent on task {task_id}: {type(e).__name__}: {e}")
|
| 484 |
+
return header, f"AGENT ERROR: {type(e).__name__}: {e}"
|
| 485 |
+
|
| 486 |
+
|
| 487 |
def run_and_submit_all(profile: gr.OAuthProfile | None):
|
| 488 |
"""
|
| 489 |
Fetches all questions, runs the GAIAAgent on them (downloading any
|
|
|
|
| 547 |
file_path = download_task_file(api_url, task_id, file_name)
|
| 548 |
|
| 549 |
try:
|
| 550 |
+
submitted_answer = agent(question_text, file_path=file_path, file_name=file_name)
|
| 551 |
answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
|
| 552 |
results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})
|
| 553 |
except Exception as e:
|
| 554 |
+
print(f"Error running agent on task {task_id}: {type(e).__name__}: {e}")
|
| 555 |
results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": f"AGENT ERROR: {e}"})
|
| 556 |
|
| 557 |
if not answers_payload:
|
|
|
|
| 634 |
outputs=[status_output, results_table]
|
| 635 |
)
|
| 636 |
|
| 637 |
+
with gr.Accordion("Developer: test a single question (no submission)", open=False):
|
| 638 |
+
gr.Markdown(
|
| 639 |
+
"Leave the box empty to pull a random task from `/random-question`, "
|
| 640 |
+
"or paste a specific `task_id` from `/questions`."
|
| 641 |
+
)
|
| 642 |
+
single_task_id = gr.Textbox(label="task_id (optional)", placeholder="leave blank for a random question")
|
| 643 |
+
single_button = gr.Button("Test One Question")
|
| 644 |
+
single_question_output = gr.Textbox(label="Task / Question", lines=8, interactive=False)
|
| 645 |
+
single_answer_output = gr.Textbox(label="Agent Answer (cleaned)", lines=3, interactive=False)
|
| 646 |
+
|
| 647 |
+
single_button.click(
|
| 648 |
+
fn=run_single_question,
|
| 649 |
+
inputs=[single_task_id],
|
| 650 |
+
outputs=[single_question_output, single_answer_output],
|
| 651 |
+
)
|
| 652 |
+
|
| 653 |
if __name__ == "__main__":
|
| 654 |
print("\n" + "-" * 30 + " App Starting " + "-" * 30)
|
| 655 |
space_host_startup = os.getenv("SPACE_HOST")
|
|
|
|
| 657 |
|
| 658 |
if space_host_startup:
|
| 659 |
print(f"✅ SPACE_HOST found: {space_host_startup}")
|
| 660 |
+
print(f" Runtime URL: https://{space_host_startup}")
|
| 661 |
else:
|
| 662 |
print("ℹ️ SPACE_HOST environment variable not found (running locally?).")
|
| 663 |
|
|
@@ -2,6 +2,7 @@ gradio
|
|
| 2 |
spaces
|
| 3 |
requests
|
| 4 |
pandas
|
| 5 |
-
smolagents[toolkit]
|
|
|
|
| 6 |
pypdf
|
| 7 |
-
openpyxl
|
|
|
|
| 2 |
spaces
|
| 3 |
requests
|
| 4 |
pandas
|
| 5 |
+
smolagents[toolkit]==1.26.0
|
| 6 |
+
openai
|
| 7 |
pypdf
|
| 8 |
+
openpyxl
|