Upload folder using huggingface_hub

.env.example

@@ -17,7 +17,7 @@ HF_TOKEN=hf_your_token_here
 
 # ── Optional overrides ────────────────────────────────────────────────────
 # LOCAL_IMAGE_NAME=nl2sql-bench:latest    # Docker image name for local dev
-# SPACE_URL=https://your-space.hf.space  # Deployed HF Space URL
+SPACE_URL=https://your-space.hf.space  # Deployed HF Space URL
 # NL2SQL_DEFAULT_TASK=simple-filter      # Default task (overridden per episode)
 # NL2SQL_MAX_STEPS=5                     # Max steps per episode
 # ENABLE_WEB_INTERFACE=true              # Enable /web UI for debugging

client.py

@@ -4,10 +4,12 @@ import os
 from typing import Any, Dict, Optional
 from dataclasses import dataclass
 
+
 @dataclass
 class NL2SQLAction:
     query: str
 
+
 @dataclass
 class NL2SQLObservation:
     question: str
@@ -23,16 +25,18 @@ class NL2SQLObservation:
     reward: float
     score: float
 
+
 @dataclass
 class StepResult:
     observation: NL2SQLObservation
     reward: float
     done: bool
 
+
 class NL2SQLEnv:
     def __init__(self, base_url: str = "http://localhost:8000"):
-        self.base_url = base_url
-        self.client = httpx.AsyncClient(base_url=base_url, timeout=60.0)
+        self.base_url = base_url.rstrip("/")
+        self.client = httpx.AsyncClient(base_url=self.base_url, timeout=120.0)
 
     async def __aenter__(self):
         return self
@@ -42,23 +46,32 @@ class NL2SQLEnv:
 
     async def reset(self) -> StepResult:
         task_name = os.getenv("NL2SQL_DEFAULT_TASK", "simple-filter")
-        resp = await self.client.post("/reset", json={"task_name": task_name})
+        # Send task_name both ways — some openenv-core versions read from body,
+        # some from the action wrapper. Belt-and-suspenders.
+        payload = {"task_name": task_name}
+        resp = await self.client.post("/reset", json=payload)
+        resp.raise_for_status()
         return self._parse_result(resp.json())
 
     async def step(self, action: NL2SQLAction) -> StepResult:
-        payload = {"query": action.query}
+        # CRITICAL FIX: The server's action_cls=NL2SQLAction expects the payload
+        # wrapped in {"action": {"query": ...}} per OpenEnv protocol.
+        # Sending {"query": ...} at the top level bypasses action parsing → 0 reward.
+        payload = {"action": {"query": action.query}}
         resp = await self.client.post("/step", json=payload)
+        resp.raise_for_status()
         return self._parse_result(resp.json())
 
     def _parse_result(self, payload: Dict[str, Any]) -> StepResult:
         obs_data = payload.get("observation", payload)
-        
-        # ── THE BULLETPROOF REWARD EXTRACTOR ──
-        # Check both the top-level payload and the nested observation dict.
-        val1 = float(payload.get("reward") or 0.0)
-        val2 = float(obs_data.get("reward") or 0.0)
-        safe_reward = max(val1, val2)
-        
+
+        # Extract reward — check top-level payload first (OpenEnv puts it there),
+        # then fall back to nested observation dict.
+        raw_reward = payload.get("reward")
+        if raw_reward is None:
+            raw_reward = obs_data.get("reward")
+        safe_reward = float(raw_reward) if raw_reward is not None else 0.0
+
         safe_score = float(obs_data.get("score") or 0.0)
         safe_done = bool(payload.get("done") or obs_data.get("done") or False)
 

debug_server.py

@@ -0,0 +1,21 @@
+import requests
+
+space_url = "https://ritvik360-nl2sql-bench.hf.space"
+
+print("1. Resetting environment...")
+try:
+    res_reset = requests.post(f"{space_url}/reset", json={"task_name": "simple-filter"})
+    print(f"Reset Status: {res_reset.status_code}")
+except Exception as e:
+    print(f"Network error on reset: {e}")
+
+print("\n2. Sending test SQL step...")
+try:
+    # We send a perfectly valid query that should score > 0
+    payload = {"query": "SELECT id, name, email, country FROM customers WHERE tier = 'gold'"}
+    res_step = requests.post(f"{space_url}/step", json=payload)
+    
+    print(f"Step Status: {res_step.status_code}")
+    print(f"Step Response:\n{res_step.text}")
+except Exception as e:
+    print(f"Network error on step: {e}")

inference.py

@@ -6,7 +6,7 @@ MANDATORY COMPLIANCE
 --------------------
 - Named `inference.py`, placed in project root.
 - Uses OpenAI client for all LLM calls.
-- Reads: API_BASE_URL, MODEL_NAME, HF_TOKEN from environment.
+- Reads: API_BASE_URL, MODEL_NAME, API_KEY (+ HF_TOKEN fallback) from environment.
 - Emits [START] / [STEP] / [END] lines to stdout in the exact format below.
 - Runs all 3 tasks; total runtime < 20 min on 2 vCPU / 8 GB.
 
@@ -27,39 +27,38 @@ from typing import List, Optional
 
 from openai import OpenAI
 
-# # ── Configuration ──────────────────────────────────────────────────────────
-# API_BASE_URL   = os.getenv("API_BASE_URL", "https://router.huggingface.co/v1")
-# MODEL_NAME     = os.getenv("MODEL_NAME",   "Qwen/Qwen2.5-7B-Instruct")
-# API_KEY        = os.getenv("HF_TOKEN") or os.getenv("OPENAI_API_KEY", "")
-# IMAGE_NAME     = os.getenv("LOCAL_IMAGE_NAME", "nl2sql-bench:latest")
-# SPACE_URL      = os.getenv("SPACE_URL", "http://localhost:8000")
+# ── Configuration ──────────────────────────────────────────────────────────
+# CRITICAL: API_BASE_URL and API_KEY are injected by the competition evaluator.
+# Do NOT hardcode values. The evaluator injects their LiteLLM proxy URL + key.
+API_BASE_URL = os.getenv("API_BASE_URL", "https://router.huggingface.co/v1")
 
-# BENCHMARK      = "nl2sql-bench"
-# MAX_STEPS      = 5
-# TEMPERATURE    = 0.2      # Low temp for SQL generation
-# MAX_TOKENS     = 512
-# SUCCESS_THRESHOLD = 0.7   # score >= 0.7 → success
+# CRITICAL FIX: Default MODEL_NAME must be a model available on the HF router /
+# the competition's LiteLLM proxy. "ritvik360/qwen-7b-nl2sql-merged_1" is NOT
+# on their proxy — it would silently fail and produce SELECT 1 for all steps.
+# The competition injects MODEL_NAME if they want to override it.
+MODEL_NAME = os.getenv("MODEL_NAME", "Qwen/Qwen2.5-7B-Instruct")
 
-# TASKS = ["simple-filter", "join-aggregation", "analytics-window"]
+# CRITICAL FIX: Read API_KEY first (competition injects this), then fall back
+# to HF_TOKEN. Both variable names must be checked.
+API_KEY = os.getenv("API_KEY") or os.getenv("HF_TOKEN", "") or os.getenv("OPENAI_API_KEY", "")
 
-# ── Configuration ──────────────────────────────────────────────────────────
-API_BASE_URL      = os.getenv("API_BASE_URL", "https://router.huggingface.co/v1")
-# Points to your newly uploaded fine-tuned weights!
-MODEL_NAME        = os.getenv("MODEL_NAME", "ritvik360/qwen-7b-nl2sql-merged_1") 
-# CRITICAL FIX: Looks for 'API_KEY' first to satisfy the evaluator's LiteLLM proxy
-API_KEY           = os.getenv("API_KEY")  or os.getenv("HF_TOKEN", "") or os.getenv("OPENAI_API_KEY")
-IMAGE_NAME        = os.getenv("LOCAL_IMAGE_NAME", "nl2sql-bench:latest")
-# CRITICAL FIX: Point the default directly to your live HF Space!
 SPACE_URL         = os.getenv("SPACE_URL", "https://ritvik360-nl2sql-bench.hf.space")
+IMAGE_NAME        = os.getenv("LOCAL_IMAGE_NAME", "nl2sql-bench:latest")
 
 BENCHMARK         = "nl2sql-bench"
 MAX_STEPS         = 5
-TEMPERATURE       = 0.2      # Low temp for SQL generation
+TEMPERATURE       = 0.2
 MAX_TOKENS        = 512
-SUCCESS_THRESHOLD = 0.7      # score >= 0.7 → success
+SUCCESS_THRESHOLD = 0.7
 
 TASKS = ["simple-filter", "join-aggregation", "analytics-window"]
 
+# ── Startup diagnostics (stderr — not scored) ─────────────────────────────
+print(f"[DEBUG] API_BASE_URL = {API_BASE_URL}", file=sys.stderr, flush=True)
+print(f"[DEBUG] MODEL_NAME   = {MODEL_NAME}", file=sys.stderr, flush=True)
+print(f"[DEBUG] API_KEY set  = {bool(API_KEY)} (len={len(API_KEY)})", file=sys.stderr, flush=True)
+print(f"[DEBUG] SPACE_URL    = {SPACE_URL}", file=sys.stderr, flush=True)
+
 # ── System prompt ──────────────────────────────────────────────────────────
 SYSTEM_PROMPT = textwrap.dedent("""
 You are an expert SQL analyst working with a SQLite e-commerce database.
@@ -95,7 +94,6 @@ def log_start(task: str, model: str) -> None:
 def log_step(
     step: int, action: str, reward: float, done: bool, error: Optional[str]
 ) -> None:
-    # Collapse multi-line SQL to single line for log compliance
     action_single = " ".join(action.split())
     error_val = error.replace("\n", " ") if error else "null"
     print(
@@ -148,7 +146,14 @@ def build_user_prompt(
 
 
 def call_llm(client: OpenAI, user_prompt: str) -> str:
+    # CRITICAL: Do NOT silently swallow exceptions with a bare `except Exception`.
+    # Silent failure means inference.py "succeeds" but makes zero LLM API calls,
+    # which causes the competition's LLM Criteria Check to fail.
     try:
+        print(
+            f"[DEBUG] Calling LLM: model={MODEL_NAME} base_url={API_BASE_URL}",
+            file=sys.stderr, flush=True
+        )
         resp = client.chat.completions.create(
             model=MODEL_NAME,
             messages=[
@@ -160,6 +165,7 @@ def call_llm(client: OpenAI, user_prompt: str) -> str:
             stream=False,
         )
         text = (resp.choices[0].message.content or "").strip()
+        print(f"[DEBUG] LLM raw response (first 120 chars): {text[:120]}", file=sys.stderr, flush=True)
         # Strip markdown code fences if model wraps in ```sql ... ```
         if text.startswith("```"):
             lines = text.split("\n")
@@ -169,8 +175,11 @@ def call_llm(client: OpenAI, user_prompt: str) -> str:
             ).strip()
         return text if text else "SELECT 1"
     except Exception as exc:
-        print(f"[DEBUG] LLM call failed: {exc}", file=sys.stderr, flush=True)
-        return "SELECT 1"
+        # Log the full error — this is the signal that tells you what went wrong
+        print(f"[DEBUG] LLM call FAILED: {type(exc).__name__}: {exc}", file=sys.stderr, flush=True)
+        # Re-raise so the episode is marked failed, not silently scored as 0.
+        # A visible failure is better than a silent one that breaks the LLM check.
+        raise
 
 
 # ── Single-task episode ────────────────────────────────────────────────────
@@ -185,11 +194,7 @@ async def run_task(client: OpenAI, env, task_name: str) -> dict:
     log_start(task_name, MODEL_NAME)
 
     try:
-        # Reset — pass task_name via action payload or query param
-        # OpenEnv reset() may not accept task args via HTTP; we rely on
-        # NL2SQL_DEFAULT_TASK env-var being set before calling, OR we
-        # pass it as a reset parameter if the server supports it.
-        result = await env.reset() # changed
+        result = await env.reset()
         obs = result.observation
 
         for step in range(1, MAX_STEPS + 1):
@@ -208,7 +213,7 @@ async def run_task(client: OpenAI, env, task_name: str) -> dict:
 
             sql = call_llm(client, user_prompt)
 
-            from models import NL2SQLAction  # local to avoid circular at module level
+            from models import NL2SQLAction
             action = NL2SQLAction(query=sql)
             result = await env.step(action)
             obs = result.observation
@@ -225,13 +230,12 @@ async def run_task(client: OpenAI, env, task_name: str) -> dict:
             if done:
                 break
 
-        # Compute final score
         score   = sum(rewards) / max(len(rewards), 1)
         score   = round(min(max(score, 0.0), 1.0), 4)
         success = score >= SUCCESS_THRESHOLD
 
     except Exception as exc:
-        print(f"[DEBUG] Episode error for {task_name}: {exc}", file=sys.stderr, flush=True)
+        print(f"[DEBUG] Episode error for {task_name}: {type(exc).__name__}: {exc}", file=sys.stderr, flush=True)
     finally:
         log_end(success=success, steps=steps_taken, score=score, rewards=rewards)
 
@@ -241,17 +245,21 @@ async def run_task(client: OpenAI, env, task_name: str) -> dict:
 # ── Main ───────────────────────────────────────────────────────────────────
 
 async def main() -> None:
+    # Validate that API_KEY is present — fail fast with a clear message
+    if not API_KEY:
+        print(
+            "[ERROR] No API key found. Set API_KEY or HF_TOKEN environment variable.",
+            file=sys.stderr, flush=True
+        )
+        sys.exit(1)
+
     client = OpenAI(base_url=API_BASE_URL, api_key=API_KEY)
 
-    # Import here to avoid import errors if openenv not installed during lint
     from client import NL2SQLEnv
 
     all_results = []
 
     for task_name in TASKS:
-        # Set the default task for the server session via env-var approach.
-        # For the hosted Space, we rely on the task cycling implemented in
-        # the task registry's round-robin iterator.
         os.environ["NL2SQL_DEFAULT_TASK"] = task_name
 
         try:
@@ -260,20 +268,18 @@ async def main() -> None:
                 all_results.append(result)
         except Exception as exc:
             print(
-                f"[DEBUG] Failed to connect for task {task_name}: {exc}",
+                f"[DEBUG] Failed to connect for task {task_name}: {type(exc).__name__}: {exc}",
                 file=sys.stderr,
                 flush=True,
             )
-            # Emit a zero-score END to keep log format valid
             log_end(success=False, steps=0, score=0.0, rewards=[])
             all_results.append({"task": task_name, "success": False, "score": 0.0})
 
-    # Summary to stderr (not scored, for human readability)
+    # Summary to stderr
     print("\n=== Baseline Summary ===", file=sys.stderr)
     for r in all_results:
         print(
-            f"  {r['task']:20s}  score={r['score']:.3f}  "
-            f"success={r['success']}",
+            f"  {r['task']:20s}  score={r['score']:.3f}  success={r['success']}",
             file=sys.stderr,
         )
     avg = sum(r["score"] for r in all_results) / max(len(all_results), 1)
@@ -281,4 +287,4 @@ async def main() -> None:
 
 
 if __name__ == "__main__":
-    asyncio.run(main())
+    asyncio.run(main())