Upload folder using huggingface_hub

Dockerfile

@@ -0,0 +1,81 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+# Multi-stage build using openenv-base
+# This Dockerfile is flexible and works for both:
+# - In-repo environments (with local OpenEnv sources)
+# - Standalone environments (with openenv from PyPI/Git)
+# The build script (openenv build) handles context detection and sets appropriate build args.
+
+ARG BASE_IMAGE=ghcr.io/meta-pytorch/openenv-base:latest
+FROM ${BASE_IMAGE} AS builder
+
+WORKDIR /app
+
+# Ensure git is available (required for installing dependencies from VCS)
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends git && \
+    rm -rf /var/lib/apt/lists/*
+
+# Build argument to control whether we're building standalone or in-repo
+ARG BUILD_MODE=in-repo
+ARG ENV_NAME=finenv
+
+# Copy environment code (always at root of build context)
+COPY . /app/env
+
+# For in-repo builds, openenv is already vendored in the build context
+# For standalone builds, openenv will be installed via pyproject.toml
+WORKDIR /app/env
+
+# Ensure uv is available (for local builds where base image lacks it)
+RUN if ! command -v uv >/dev/null 2>&1; then \
+        curl -LsSf https://astral.sh/uv/install.sh | sh && \
+        mv /root/.local/bin/uv /usr/local/bin/uv && \
+        mv /root/.local/bin/uvx /usr/local/bin/uvx; \
+    fi
+    
+# Install dependencies using uv sync
+# If uv.lock exists, use it; otherwise resolve on the fly
+RUN --mount=type=cache,target=/root/.cache/uv \
+    if [ -f uv.lock ]; then \
+        uv sync --frozen --no-install-project --no-editable; \
+    else \
+        uv sync --no-install-project --no-editable; \
+    fi
+
+RUN --mount=type=cache,target=/root/.cache/uv \
+    if [ -f uv.lock ]; then \
+        uv sync --frozen --no-editable; \
+    else \
+        uv sync --no-editable; \
+    fi
+
+# Final runtime stage
+FROM ${BASE_IMAGE}
+
+WORKDIR /app
+
+# Copy the virtual environment from builder
+COPY --from=builder /app/env/.venv /app/.venv
+
+# Copy the environment code
+COPY --from=builder /app/env /app/env
+
+# Set PATH to use the virtual environment
+ENV PATH="/app/.venv/bin:$PATH"
+
+# Set PYTHONPATH so imports work correctly
+ENV PYTHONPATH="/app/env:$PYTHONPATH"
+
+# Health check
+HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
+    CMD curl -f http://localhost:8000/health || exit 1
+
+# Run the FastAPI server
+# The module path is constructed to work with the /app/env structure
+ENV ENABLE_WEB_INTERFACE=true
+CMD ["sh", "-c", "cd /app/env && uvicorn server.app:app --host 0.0.0.0 --port 8000"]

README.md

@@ -1,10 +1,255 @@
 ---
-title: Finenv
-emoji: 🏆
-colorFrom: gray
-colorTo: yellow
+title: Finenv Environment Server
+emoji: 📻
+colorFrom: red
+colorTo: pink
 sdk: docker
 pinned: false
+app_port: 8000
+base_path: /web
+tags:
+  - openenv
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Finenv Environment
+
+A simple test environment that echoes back messages. Perfect for testing the env APIs as well as demonstrating environment usage patterns.
+
+## Quick Start
+
+The simplest way to use the Finenv environment is through the `FinenvEnv` class:
+
+```python
+from finenv import FinenvAction, FinenvEnv
+
+try:
+    # Create environment from Docker image
+    finenvenv = FinenvEnv.from_docker_image("finenv-env:latest")
+
+    # Reset
+    result = finenvenv.reset()
+    print(f"Reset: {result.observation.echoed_message}")
+
+    # Send multiple messages
+    messages = ["Hello, World!", "Testing echo", "Final message"]
+
+    for msg in messages:
+        result = finenvenv.step(FinenvAction(message=msg))
+        print(f"Sent: '{msg}'")
+        print(f"  → Echoed: '{result.observation.echoed_message}'")
+        print(f"  → Length: {result.observation.message_length}")
+        print(f"  → Reward: {result.reward}")
+
+finally:
+    # Always clean up
+    finenvenv.close()
+```
+
+That's it! The `FinenvEnv.from_docker_image()` method handles:
+- Starting the Docker container
+- Waiting for the server to be ready
+- Connecting to the environment
+- Container cleanup when you call `close()`
+
+## Building the Docker Image
+
+Before using the environment, you need to build the Docker image:
+
+```bash
+# From project root
+docker build -t finenv-env:latest -f server/Dockerfile .
+```
+
+## Deploying to Hugging Face Spaces
+
+You can easily deploy your OpenEnv environment to Hugging Face Spaces using the `openenv push` command:
+
+```bash
+# From the environment directory (where openenv.yaml is located)
+openenv push
+
+# Or specify options
+openenv push --namespace my-org --private
+```
+
+The `openenv push` command will:
+1. Validate that the directory is an OpenEnv environment (checks for `openenv.yaml`)
+2. Prepare a custom build for Hugging Face Docker space (enables web interface)
+3. Upload to Hugging Face (ensuring you're logged in)
+
+### Prerequisites
+
+- Authenticate with Hugging Face: The command will prompt for login if not already authenticated
+
+### Options
+
+- `--directory`, `-d`: Directory containing the OpenEnv environment (defaults to current directory)
+- `--repo-id`, `-r`: Repository ID in format 'username/repo-name' (defaults to 'username/env-name' from openenv.yaml)
+- `--base-image`, `-b`: Base Docker image to use (overrides Dockerfile FROM)
+- `--private`: Deploy the space as private (default: public)
+
+### Examples
+
+```bash
+# Push to your personal namespace (defaults to username/env-name from openenv.yaml)
+openenv push
+
+# Push to a specific repository
+openenv push --repo-id my-org/my-env
+
+# Push with a custom base image
+openenv push --base-image ghcr.io/meta-pytorch/openenv-base:latest
+
+# Push as a private space
+openenv push --private
+
+# Combine options
+openenv push --repo-id my-org/my-env --base-image custom-base:latest --private
+```
+
+After deployment, your space will be available at:
+`https://huggingface.co/spaces/<repo-id>`
+
+The deployed space includes:
+- **Web Interface** at `/web` - Interactive UI for exploring the environment
+- **API Documentation** at `/docs` - Full OpenAPI/Swagger interface
+- **Health Check** at `/health` - Container health monitoring
+- **WebSocket** at `/ws` - Persistent session endpoint for low-latency interactions
+
+## Environment Details
+
+### Action
+**FinenvAction**: Contains a single field
+- `message` (str) - The message to echo back
+
+### Observation
+**FinenvObservation**: Contains the echo response and metadata
+- `echoed_message` (str) - The message echoed back
+- `message_length` (int) - Length of the message
+- `reward` (float) - Reward based on message length (length × 0.1)
+- `done` (bool) - Always False for echo environment
+- `metadata` (dict) - Additional info like step count
+
+### Reward
+The reward is calculated as: `message_length × 0.1`
+- "Hi" → reward: 0.2
+- "Hello, World!" → reward: 1.3
+- Empty message → reward: 0.0
+
+## Advanced Usage
+
+### Connecting to an Existing Server
+
+If you already have a Finenv environment server running, you can connect directly:
+
+```python
+from finenv import FinenvEnv
+
+# Connect to existing server
+finenvenv = FinenvEnv(base_url="<ENV_HTTP_URL_HERE>")
+
+# Use as normal
+result = finenvenv.reset()
+result = finenvenv.step(FinenvAction(message="Hello!"))
+```
+
+Note: When connecting to an existing server, `finenvenv.close()` will NOT stop the server.
+
+### Using the Context Manager
+
+The client supports context manager usage for automatic connection management:
+
+```python
+from finenv import FinenvAction, FinenvEnv
+
+# Connect with context manager (auto-connects and closes)
+with FinenvEnv(base_url="http://localhost:8000") as env:
+    result = env.reset()
+    print(f"Reset: {result.observation.echoed_message}")
+    # Multiple steps with low latency
+    for msg in ["Hello", "World", "!"]:
+        result = env.step(FinenvAction(message=msg))
+        print(f"Echoed: {result.observation.echoed_message}")
+```
+
+The client uses WebSocket connections for:
+- **Lower latency**: No HTTP connection overhead per request
+- **Persistent session**: Server maintains your environment state
+- **Efficient for episodes**: Better for many sequential steps
+
+### Concurrent WebSocket Sessions
+
+The server supports multiple concurrent WebSocket connections. To enable this,
+modify `server/app.py` to use factory mode:
+
+```python
+# In server/app.py - use factory mode for concurrent sessions
+app = create_app(
+    FinenvEnvironment,  # Pass class, not instance
+    FinenvAction,
+    FinenvObservation,
+    max_concurrent_envs=4,  # Allow 4 concurrent sessions
+)
+```
+
+Then multiple clients can connect simultaneously:
+
+```python
+from finenv import FinenvAction, FinenvEnv
+from concurrent.futures import ThreadPoolExecutor
+
+def run_episode(client_id: int):
+    with FinenvEnv(base_url="http://localhost:8000") as env:
+        result = env.reset()
+        for i in range(10):
+            result = env.step(FinenvAction(message=f"Client {client_id}, step {i}"))
+        return client_id, result.observation.message_length
+
+# Run 4 episodes concurrently
+with ThreadPoolExecutor(max_workers=4) as executor:
+    results = list(executor.map(run_episode, range(4)))
+```
+
+## Development & Testing
+
+### Direct Environment Testing
+
+Test the environment logic directly without starting the HTTP server:
+
+```bash
+# From the server directory
+python3 server/finenv_environment.py
+```
+
+This verifies that:
+- Environment resets correctly
+- Step executes actions properly
+- State tracking works
+- Rewards are calculated correctly
+
+### Running Locally
+
+Run the server locally for development:
+
+```bash
+uvicorn server.app:app --reload
+```
+
+## Project Structure
+
+```
+finenv/
+├── .dockerignore         # Docker build exclusions
+├── __init__.py            # Module exports
+├── README.md              # This file
+├── openenv.yaml           # OpenEnv manifest
+├── pyproject.toml         # Project metadata and dependencies
+├── uv.lock                # Locked dependencies (generated)
+├── client.py              # FinenvEnv client
+├── models.py              # Action and Observation models
+└── server/
+    ├── __init__.py        # Server module exports
+    ├── finenv_environment.py  # Core environment logic
+    ├── app.py             # FastAPI application (HTTP + WebSocket endpoints)
+    └── Dockerfile         # Container image definition
+```

__init__.py

@@ -0,0 +1,16 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""Finenv Environment."""
+
+from .client import FinenvEnv
+from .models import FinenvAction, FinenvObservation
+
+__all__ = [
+    "FinenvAction",
+    "FinenvObservation",
+    "FinenvEnv",
+]

client.py

@@ -0,0 +1,108 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""Finenv Environment Client."""
+
+from typing import Dict
+
+from openenv.core import EnvClient
+from openenv.core.client_types import StepResult
+from openenv.core.env_server.types import State
+
+from .models import FinenvAction, FinenvObservation
+
+
+class FinenvEnv(
+    EnvClient[FinenvAction, FinenvObservation, State]
+):
+    """
+    Client for the Finenv Environment.
+
+    This client maintains a persistent WebSocket connection to the environment server,
+    enabling efficient multi-step interactions with lower latency.
+    Each client instance has its own dedicated environment session on the server.
+
+    Example:
+        >>> # Connect to a running server
+        >>> with FinenvEnv(base_url="http://localhost:8000") as client:
+        ...     result = client.reset()
+        ...     print(result.observation.echoed_message)
+        ...
+        ...     result = client.step(FinenvAction(message="Hello!"))
+        ...     print(result.observation.echoed_message)
+
+    Example with Docker:
+        >>> # Automatically start container and connect
+        >>> client = FinenvEnv.from_docker_image("finenv-env:latest")
+        >>> try:
+        ...     result = client.reset()
+        ...     result = client.step(FinenvAction(message="Test"))
+        ... finally:
+        ...     client.close()
+    """
+
+    def _step_payload(self, action: FinenvAction) -> Dict:
+        """
+        Convert FinenvAction to JSON payload for step message.
+
+        Args:
+            action: FinenvAction instance
+
+        Returns:
+            Dictionary representation suitable for JSON encoding
+        """
+        return {
+        "type": action.type,
+        "quantity": action.quantity,
+        "stock": action.stock,
+        "market": action.market,
+        "initial_cash": action.initial_cash,
+        "max_steps": action.max_steps,
+    }
+
+    def _parse_result(self, payload: Dict) -> StepResult[FinenvObservation]:
+        """
+        Parse server response into StepResult[FinenvObservation].
+
+        Args:
+            payload: JSON response data from server
+
+        Returns:
+            StepResult with FinenvObservation
+        """
+        obs = payload.get("observation", {})
+
+        observation = FinenvObservation(
+            stock=obs.get("stock"),
+            market=obs.get("market"),
+            price=obs.get("price"),
+            shares=obs.get("shares"),
+            cash=obs.get("cash"),
+            done=payload.get("done", False),
+            reward=payload.get("reward"),
+            metadata=obs.get("metadata", {}),
+        )
+
+        return StepResult(
+            observation=observation,
+            reward=payload.get("reward"),
+            done=payload.get("done", False),
+        )
+
+    def _parse_state(self, payload: Dict) -> State:
+        """
+        Parse server response into State object.
+
+        Args:
+            payload: JSON response from state request
+
+        Returns:
+            State object with episode_id and step_count
+        """
+        return State(
+            episode_id=payload.get("episode_id"),
+            step_count=payload.get("step_count", 0),
+        )

inference.py

@@ -0,0 +1,259 @@
+"""
+Inference Script Example
+===================================
+MANDATORY
+- Before submitting, ensure the following variables are defined in your environment configuration:
+    API_BASE_URL   The API endpoint for the LLM.
+    MODEL_NAME     The model identifier to use for inference.
+    HF_TOKEN       Your Hugging Face / API key.
+    LOCAL_IMAGE_NAME The name of the local image to use for the environment if you are using from_docker_image()
+                     method
+
+- Defaults are set only for API_BASE_URL and MODEL_NAME 
+    (and should reflect your active inference setup):
+    API_BASE_URL = os.getenv("API_BASE_URL", "<your-active-endpoint>")
+    MODEL_NAME = os.getenv("MODEL_NAME", "<your-active-model>")
+    
+- The inference script must be named `inference.py` and placed in the root directory of the project
+- Participants must use OpenAI Client for all LLM calls using above variables
+
+STDOUT FORMAT
+- The script must emit exactly three line types to stdout, in this order:
+
+    [START] task=<task_name> env=<benchmark> model=<model_name>
+    [STEP]  step=<n> action=<action_str> reward=<0.00> done=<true|false> error=<msg|null>
+    [END]   success=<true|false> steps=<n> score=<score> rewards=<r1,r2,...,rn>
+
+  Rules:
+    - One [START] line at episode begin.
+    - One [STEP] line per step, immediately after env.step() returns.
+    - One [END] line after env.close(), always emitted (even on exception).
+    - reward and rewards are formatted to 2 decimal places.
+    - done and success are lowercase booleans: true or false.
+    - error is the raw last_action_error string, or null if none.
+    - All fields on a single line with no newlines within a line.
+    - Each tasks should return score in [0, 1]
+
+  Example:
+    [START] task=click-test env=miniwob model=Qwen3-VL-30B
+    [STEP] step=1 action=click('123') reward=0.00 done=false error=null
+    [STEP] step=2 action=fill('456','text') reward=0.00 done=false error=null
+    [STEP] step=3 action=click('789') reward=1.00 done=true error=null
+    [END] success=true steps=3 score=1.00 rewards=0.00,0.00,1.00
+"""
+import asyncio
+import os
+import textwrap
+from typing import List, Optional
+
+from openai import OpenAI
+
+from finenv.client import FinenvEnv
+from finenv.models import FinenvAction
+
+# ==============================
+# ENV VARIABLES (MANDATORY)
+# ==============================
+IMAGE_NAME = os.getenv("LOCAL_IMAGE_NAME", "finenv")
+API_KEY = os.getenv("HF_TOKEN") or os.getenv("OPENAI_API_KEY")
+
+API_BASE_URL = os.getenv("API_BASE_URL")
+MODEL_NAME = os.getenv("MODEL_NAME")
+
+TASK_NAME = os.getenv("FINENV_TASK", "easy")
+BENCHMARK = "finenv"
+
+MAX_STEPS = 15
+SUCCESS_SCORE_THRESHOLD = 0.2  # normalized score in [0, 1]
+
+SYSTEM_PROMPT = textwrap.dedent(
+    """
+    You are interacting with a stock trading environment. 
+    Your goal is to maximize profit by buying, selling, or holding shares of a stock over a series of steps.
+    At each step, you can choose one of the following actions:  
+    - buy: purchase 1 share of the stock at the current price 
+    - sell: sell 1 share of the stock at the current price (only if you have shares to sell)
+    - hold: take no action
+    The environment will provide feedback in the form of rewards based on the change in your portfolio value.
+    Your objective is to achieve the highest possible return by the end of the episode.
+    """
+).strip()
+
+# ==============================
+# LOGGING (STRICT FORMAT)
+# ==============================
+def log_start(task: str, env: str, model: str) -> None:
+    print(f"[START] task={task} env={env} model={model}", flush=True)
+
+
+def log_step(step: int, action: str, reward: float, done: bool, error: Optional[str]) -> None:
+    error_val = error if error else "null"
+    done_val = str(done).lower()
+    print(
+        f"[STEP] step={step} action={action} reward={reward:.2f} done={done_val} error={error_val}",
+        flush=True,
+    )
+
+
+def log_end(success: bool, steps: int, score: float, rewards: List[float]) -> None:
+    rewards_str = ",".join(f"{r:.2f}" for r in rewards)
+    print(
+        f"[END] success={str(success).lower()} steps={steps} score={score:.3f} rewards={rewards_str}",
+        flush=True,
+    )
+
+
+# ==============================
+# MODEL DECISION LOGIC
+# ==============================
+def get_model_action(client: OpenAI, step: int, last_reward: float, history: List[str]) -> str:
+    """
+    Uses LLM to decide trading action.
+    """
+
+    prompt = f"""
+You are a trading agent.
+
+Step: {step}
+Last reward: {last_reward}
+Recent history:
+{history[-3:] if history else "None"}
+
+Choose ONE:
+buy
+sell
+hold
+
+Respond with only one word.
+"""
+
+    try:
+        response = client.chat.completions.create(
+            model=MODEL_NAME,
+            messages=[{"role": "user", "content": prompt}],
+            temperature=0.2,
+        )
+
+        action = (response.choices[0].message.content or "").strip().lower()
+
+        if action not in ["buy", "sell", "hold"]:
+            return "hold"
+
+        return action
+
+    except Exception as exc:
+        print(f"[DEBUG] Model request failed: {exc}", flush=True)
+        return "hold"
+
+
+# ==============================
+# MAIN EXECUTION
+# ==============================
+async def main() -> None:
+    client = OpenAI(base_url=API_BASE_URL, api_key=API_KEY)
+
+    env = await FinenvEnv.from_docker_image(IMAGE_NAME)
+
+    history: List[str] = []
+    rewards: List[float] = []
+
+    steps_taken = 0
+    score = 0.0
+    success = False
+
+    log_start(task=TASK_NAME, env=BENCHMARK, model=MODEL_NAME)
+
+    try:
+        # Reset environment
+        result = await env.reset()
+        last_reward = 0.0
+
+        # INIT STEP (MANDATORY for your env)
+        init_action = FinenvAction(
+            type="init",
+            stock="RELIANCE",
+            market="NSE",
+            initial_cash=10000,
+            max_steps=MAX_STEPS
+        )
+
+        result = await env.step(init_action)
+
+        # Log init step as step 0
+        log_step(
+            step=0,
+            action="init",
+            reward=0.00,
+            done=False,
+            error=None
+        )
+
+        # ==============================
+        # MAIN LOOP
+        # ==============================
+        for step in range(1, MAX_STEPS + 1):
+            if result.done:
+                break
+
+            action_type = get_model_action(client, step, last_reward, history)
+
+            action = FinenvAction(
+                type=action_type,
+                quantity=1
+            )
+
+            error = None
+
+            try:
+                result = await env.step(action)
+            except Exception as e:
+                error = str(e)
+                result = result  # keep last safe state
+
+            reward = float(result.reward or 0.0)
+            done = result.done
+
+            rewards.append(reward)
+            steps_taken = step
+            last_reward = reward
+
+            log_step(
+                step=step,
+                action=action_type,
+                reward=reward,
+                done=done,
+                error=error
+            )
+
+            history.append(f"{action_type}:{reward:.2f}")
+
+            if done:
+                break
+
+        # ==============================
+        # SCORE CALCULATION
+        # ==============================
+        if len(rewards) > 0:
+            score = sum(rewards) / len(rewards)
+        else:
+            score = 0.0
+
+        score = min(max(score, 0.0), 1.0)
+        success = score >= SUCCESS_SCORE_THRESHOLD
+
+    finally:
+        try:
+            await env.close()
+        except Exception as e:
+            print(f"[DEBUG] env.close() error: {e}", flush=True)
+
+        log_end(
+            success=success,
+            steps=steps_taken,
+            score=score,
+            rewards=rewards
+        )
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

models.py

@@ -0,0 +1,73 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""Data models for the Finenv environment."""
+
+from enum import Enum
+from typing import Any, Dict, Optional
+
+from openenv.core.env_server.types import Action, Observation
+
+TASKS = {
+    "easy": {
+        "goal": "Make profit in 5 steps",
+        "target_profit": 0.01
+    },
+    "medium": {
+        "goal": "Maximize portfolio value",
+        "target_profit": 0.05
+    },
+    "hard": {
+        "goal": "Achieve 10% return",
+        "target_return": 0.1
+    }
+}
+
+class StockExchangeMarket(Enum):
+    BSE = "bse"
+    NSE = "nse"
+
+class task_type(str, Enum):
+    Easy = "easy"
+    Medium = "medium"
+    Hard = "hard"
+
+class FinenvAction(Action):
+    """
+    Action model for Finenv environment.
+
+    Supports:
+    - init → initialize environment
+    - buy → buy shares
+    - sell → sell shares
+    - hold → no action
+    """
+
+    type: str  # "init", "buy", "sell", "hold"
+
+    # Trading fields
+    quantity: Optional[int] = 0
+
+    task_type: Optional[task_type] = None
+
+    # Initialization fields
+    stock: Optional[str] = None
+    market: Optional[str] = None
+    initial_cash: Optional[float] = None
+    max_steps: Optional[int] = None
+
+
+class FinenvObservation(Observation):
+    """Observation from the Finenv environment - the echoed message."""
+    stock: Optional[str] = None
+    market: Optional[StockExchangeMarket] = None
+    shares: int
+    cash: float
+    price: float
+    reward: str
+    done: bool
+    metadata: Optional[Dict[str, Any]] = None
+    

openenv.yaml

@@ -0,0 +1,14 @@
+spec_version: 1
+name: finenv
+description: Stock trading environment using real NSE/BSE data
+type: space
+runtime: fastapi
+app: server.app:app
+port: 8000
+tasks:
+  - name: easy
+    description: Achieve 1% return
+  - name: medium
+    description: Achieve 5% return
+  - name: hard
+    description: Achieve 10% return

pyproject.toml

@@ -0,0 +1,46 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+[build-system]
+requires = ["setuptools>=45", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "openenv-finenv"
+version = "0.1.0"
+description = "Finenv environment for OpenEnv"
+requires-python = ">=3.10"
+dependencies = [
+    # Core OpenEnv runtime (provides FastAPI server + HTTP client types)
+    # install from github
+    # "openenv-core[core] @ git+https://github.com/meta-pytorch/OpenEnv.git",
+    "openenv-core[core]>=0.2.2",
+    "yfinance>=0.2.0"
+    # Environment-specific dependencies
+    # Add all dependencies needed for your environment here
+    # Examples:
+    # "numpy>=1.19.0",
+    # "torch>=2.0.0",
+    # "gymnasium>=0.29.0",
+    # "openspiel>=1.0.0",
+    # "smolagents>=1.22.0,<2",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=8.0.0",
+    "pytest-cov>=4.0.0",
+]
+
+[project.scripts]
+# Server entry point - enables running via: uv run --project . server
+# or: python -m finenv.server.app
+server = "finenv.server.app:main"
+
+[tool.setuptools]
+include-package-data = true
+packages = ["finenv", "finenv.server"]
+package-dir = { "finenv" = ".", "finenv.server" = "server" }

server/__init__.py

@@ -0,0 +1,11 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""Finenv environment server components."""
+
+from .finenv_environment import FinenvEnvironment
+
+__all__ = ["FinenvEnvironment"]

server/app.py

@@ -0,0 +1,84 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+FastAPI application for the Finenv Environment.
+
+This module creates an HTTP server that exposes the FinenvEnvironment
+over HTTP and WebSocket endpoints, compatible with EnvClient.
+
+Endpoints:
+    - POST /reset: Reset the environment
+    - POST /step: Execute an action
+    - GET /state: Get current environment state
+    - GET /schema: Get action/observation schemas
+    - WS /ws: WebSocket endpoint for persistent sessions
+
+Usage:
+    # Development (with auto-reload):
+    uvicorn server.app:app --reload --host 0.0.0.0 --port 8000
+
+    # Production:
+    uvicorn server.app:app --host 0.0.0.0 --port 8000 --workers 4
+
+    # Or run directly:
+    python -m server.app
+"""
+
+try:
+    from openenv.core.env_server.http_server import create_app
+except Exception as e:  # pragma: no cover
+    raise ImportError(
+        "openenv is required for the web interface. Install dependencies with '\n    uv sync\n'"
+    ) from e
+
+try:
+    from ..models import FinenvAction, FinenvObservation
+    from .finenv_environment import FinenvEnvironment
+except ModuleNotFoundError:
+    from models import FinenvAction, FinenvObservation
+    from server.finenv_environment import FinenvEnvironment
+
+
+# Create the app with web interface and README integration
+app = create_app(
+    FinenvEnvironment,
+    FinenvAction,
+    FinenvObservation,
+    env_name="finenv",
+    max_concurrent_envs=1,  # increase this number to allow more concurrent WebSocket sessions
+)
+
+
+def main(host: str = "0.0.0.0", port: int = 8000):
+    """
+    Entry point for direct execution via uv run or python -m.
+
+    This function enables running the server without Docker:
+        uv run --project . server
+        uv run --project . server --port 8001
+        python -m finenv.server.app
+
+    Args:
+        host: Host address to bind to (default: "0.0.0.0")
+        port: Port number to listen on (default: 8000)
+
+    For production deployments, consider using uvicorn directly with
+    multiple workers:
+        uvicorn finenv.server.app:app --workers 4
+    """
+    import uvicorn
+
+    uvicorn.run(app, host=host, port=port)
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--port", type=int, default=8000)
+    args = parser.parse_args()
+    main()

server/finenv_environment.py

@@ -0,0 +1,199 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+"""
+Finenv Environment Implementation (Dynamic Stock via First Action)
+
+This environment simulates stock trading.
+The stock and market are NOT fixed — they are initialized dynamically
+using the first action of type "init".
+
+Actions supported:
+- init → initialize stock, market, capital
+- buy → buy shares
+- sell → sell shares
+- hold → do nothing
+"""
+
+from uuid import uuid4
+
+from openenv.core.env_server.interfaces import Environment
+from openenv.core.env_server.types import State
+try:
+    from ..models import FinenvAction, FinenvObservation, StockExchangeMarket
+    from .reward import reward_message
+    from .stock_price import get_stock_price
+except ImportError:
+    from models import FinenvAction, FinenvObservation, StockExchangeMarket
+    from reward import reward_message
+    from stock_price import get_stock_price
+
+
+class FinenvEnvironment(Environment):
+
+    SUPPORTS_CONCURRENT_SESSIONS: bool = True
+
+    def __init__(self):
+        """
+        Initialize environment with default values.
+        Actual stock configuration will be done using 'init' action.
+        """
+
+        self._state = State(episode_id=str(uuid4()), step_count=0)
+
+        # Default placeholders (will be overwritten)
+        self.stock = None
+        self.market = None
+        self.initial_cash = 10000.0
+        self.max_steps = 100
+        self.task = "easy"
+
+        self.cash = self.initial_cash
+        self.shares = 0
+
+        self.price_history = []
+        self.stock_price = 0.0
+
+        # Flag to check if environment is initialized
+        self.initialized = False
+
+    def reset(self) -> FinenvObservation:
+        """
+        Reset environment WITHOUT changing stock.
+
+        Stock will be set using first action (type="init")
+        """
+
+        self._state = State(episode_id=str(uuid4()), step_count=0)
+
+        self.cash = self.initial_cash
+        self.shares = 0
+
+        return FinenvObservation(
+            stock=self.stock,
+            market=self.market,
+            price=self.stock_price,
+            shares=self.shares,
+            cash=self.cash,
+            done=False,
+            reward=reward_message(0),
+            metadata={"message": "Send init action to start trading"}
+        )
+
+    def step(self, action: FinenvAction) -> FinenvObservation:
+        """
+        Execute one step.
+
+        First action MUST be:
+        {
+            "type": "init",
+            "stock": "RELIANCE",
+            "market": "NSE"
+        }
+        """
+
+        
+        if action.type == "init":
+            self.stock = action.stock
+            self.market = StockExchangeMarket(action.market.lower())
+
+            self.initial_cash = action.initial_cash or 10000.0
+            self.max_steps = action.max_steps or 100
+            self.task = action.task_type.value if action.task_type else "easy"
+
+            self.cash = self.initial_cash
+            self.shares = 0
+
+            # Fetch stock data
+            self.price_history = get_stock_price(
+                self.stock,
+                self.market.value,
+                self.max_steps
+            )
+
+            if not self.price_history:
+                raise ValueError("Invalid stock or no data available")
+
+            self.stock_price = self.price_history[0]
+
+            self.initialized = True
+
+            return FinenvObservation(
+                stock=self.stock,
+                market=self.market,
+                price=self.stock_price,
+                shares=self.shares,
+                cash=self.cash,
+                done=False,
+                reward=reward_message(0),
+                metadata={"message": "Environment initialized"}
+            )
+
+        
+        if not self.initialized:
+            raise ValueError("Environment not initialized. Send init action first.")
+
+        # Increment step count
+        self._state.step_count += 1
+
+        
+        old_value = self.cash + self.shares * self.stock_price
+
+        
+        if action.type == "buy":
+            cost = action.quantity * self.stock_price
+            if self.cash >= cost:
+                self.cash -= cost
+                self.shares += action.quantity
+
+        elif action.type == "sell":
+            if self.shares >= action.quantity:
+                self.cash += action.quantity * self.stock_price
+                self.shares -= action.quantity
+
+        # hold → no action
+
+       
+        index = self._state.step_count % len(self.price_history)
+        self.stock_price = self.price_history[index]
+
+        new_value = self.cash + self.shares * self.stock_price
+        reward = (new_value - old_value) / self.initial_cash
+        reward = max(min(reward, 1.0), -1.0)
+
+       
+        done = self._state.step_count >= self.max_steps
+        if done:
+            final_value = self.cash + self.shares * self.stock_price
+            profit = final_value - self.initial_cash
+
+            if self.task == "easy":
+                score = min(max(profit / 0.01*self.initial_cash, 0), 1)
+
+            elif self.task == "medium":
+                score = min(max(profit / 0.05*self.initial_cash, 0), 1)
+
+            elif self.task == "hard":
+                score = min(max((profit / self.initial_cash) / 0.1, 0), 1)
+
+            reward = score
+        
+        return FinenvObservation(
+            stock=self.stock,
+            market=self.market,
+            shares=self.shares,
+            cash=self.cash,
+            price=self.stock_price,
+            done=done,
+            reward=reward_message(reward),
+            metadata={
+                "step": self._state.step_count,
+                "portfolio_value": new_value
+            },
+        )
+
+    @property
+    def state(self) -> State:
+        return self._state

server/requirements.txt

@@ -0,0 +1,7 @@
+openenv[core]>=0.2.0
+fastapi>=0.115.0
+uvicorn>=0.24.0
+yfinance>=0.2.0
+
+
+

server/reward.py

@@ -0,0 +1,122 @@
+"""This is a simple reward function for the Finenv environment. 
+It developes a sentence based in the reward value.
+Positive rewards generate encouraging messages, negative rewards generate constructive feedback, and zero rewards generate neutral prompts to motivate improvement."""
+
+
+import random
+
+from sympy import Integer
+
+import random
+
+def reward_message(num:float) -> str:
+    positive_msgs = [
+        f"Awesome work! You scored {num}, keep pushing forward!",
+        f"Fantastic! {num} shows your great progress!",
+        f"Great going! {num} is a strong positive number!",
+        f"Brilliant! You achieved {num}, success is yours!",
+        f"Excellent! {num} reflects your hard work!",
+        f"Superb! A score of {num} is impressive!",
+        f"Keep it up! {num} is a great result!",
+        f"Well done! {num} shows you're improving!",
+        f"Amazing! {num} is a fantastic score!",
+        f"Outstanding! {num} proves your capability!",
+        f"Nice work! {num} is a positive step!",
+        f"Incredible! {num} shows your potential!",
+        f"Strong performance! {num} looks great!",
+        f"You're doing great! {num} is excellent!",
+        f"Fantastic effort! {num} is rewarding!",
+        f"Impressive! {num} shows growth!",
+        f"Great job! {num} is a win!",
+        f"Positive vibes! {num} is awesome!",
+        f"Keep shining! {num} is brilliant!",
+        f"Top work! {num} is fantastic!",
+        f"Excellent progress with {num}!",
+        f"Winning moment! {num} looks great!",
+        f"You nailed it with {num}!",
+        f"{num} is a sign of success!",
+        f"Keep rising! {num} is strong!",
+        f"That’s a solid {num}!",
+        f"You're on fire with {num}!",
+        f"Powerful result: {num}!",
+        f"You're unstoppable at {num}!",
+        f"Success shines in {num}!",
+        f"{num} is a proud achievement!",
+        f"Great energy with {num}!",
+        f"Keep winning with {num}!",
+        f"Victory feels like {num}!",
+        f"Big win: {num}!",
+        f"You're excelling at {num}!",
+        f"That’s impressive: {num}!",
+        f"Keep thriving with {num}!",
+        f"{num} is a milestone!",
+        f"Wonderful result: {num}!",
+        f"You're progressing with {num}!",
+        f"Sharp performance: {num}!",
+        f"Keep growing: {num}!",
+        f"That’s a boost: {num}!",
+        f"You're achieving big with {num}!",
+        f"Great momentum: {num}!",
+        f"That’s a bright {num}!",
+        f"You’re crushing it with {num}!",
+        f"Keep soaring: {num}!",
+        f"Fantastic number: {num}!"
+    ]
+
+    zero_msgs = [
+        "Your score is 0, try to move forward.",
+        "0 means no progress, take action.",
+        "You got 0, push for improvement.",
+        "0 feels neutral, aim higher.",
+        "Stuck at 0, time to grow.",
+        "0 is just a starting point.",
+        "Move ahead from 0.",
+        "0 shows no change, act now.",
+        "Break the 0 barrier.",
+        "0 needs momentum.",
+        "Rise above 0.",
+        "0 is not the end.",
+        "Push beyond 0.",
+        "0 needs effort.",
+        "Start fresh from 0.",
+        "0 is a reset point.",
+        "Climb up from 0.",
+        "0 is a pause, not stop.",
+        "Turn 0 into progress.",
+        "0 needs a push.",
+        "Go beyond 0.",
+        "0 is neutral ground.",
+        "Advance from 0.",
+        "0 needs action.",
+        "Step up from 0.",
+        "0 is just beginning.",
+        "Grow past 0.",
+        "0 needs direction.",
+        "Break out of 0.",
+        "0 is waiting for effort.",
+        "Push ahead from 0.",
+        "0 is your base.",
+        "Move upward from 0.",
+        "0 needs energy.",
+        "Turn 0 positive.",
+        "0 is not enough.",
+        "Climb higher than 0.",
+        "0 needs change.",
+        "Start moving from 0.",
+        "0 is temporary.",
+        "Build from 0.",
+        "0 needs drive.",
+        "Progress after 0.",
+        "0 is your launch point.",
+        "0 needs momentum.",
+        "Don't stay at 0.",
+        "0 is a fresh start.",
+        "Act beyond 0.",
+        "0 needs growth.",
+        "Move forward from 0."
+    ]
+
+    if num == 0:
+        return random.choice(zero_msgs)
+    elif num > 0:
+        return random.choice(positive_msgs)

server/stock_price.py

@@ -0,0 +1,29 @@
+"""Fetch stock price data for a given symbol and exchange."""
+
+import yfinance as yf
+
+
+def get_stock_price(symbol, exchange, days=None):
+    try:
+        exchange_normalized = str(exchange).lower()
+        if exchange_normalized == "nse":
+            ticker = symbol + ".NS"
+        elif exchange_normalized == "bse":
+            ticker = symbol + ".BO"
+        else:
+            return []
+
+        stock = yf.Ticker(ticker)
+        data = stock.history(period="1d") if days is None else stock.history(period=f"{days}d")
+
+        if data.empty:
+            return []
+
+        prices = data['Close'].tolist()
+        
+        return prices
+
+    except Exception:
+        return []
+    
+# print(get_stock_price("RELIANCE","NSE",days=5))

tutorial/01-environments.md

@@ -0,0 +1,1260 @@
+# OpenEnv: Production RL Made Simple
+
+<div align="center">
+
+<img src="https://upload.wikimedia.org/wikipedia/commons/1/10/PyTorch_logo_icon.svg" width="200" alt="PyTorch">
+
+### *From "Hello World" to RL Training in 5 Minutes* ✨
+
+---
+
+**What if RL environments were as easy to use as REST APIs?**
+
+That's OpenEnv. Type-safe. Isolated. Production-ready. 🎯
+
+[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/meta-pytorch/OpenEnv/blob/main/examples/OpenEnv_Tutorial.ipynb)
+[![GitHub](https://img.shields.io/badge/GitHub-meta--pytorch%2FOpenEnv-blue?logo=github)](https://github.com/meta-pytorch/OpenEnv)
+[![License](https://img.shields.io/badge/License-BSD%203--Clause-green.svg)](https://opensource.org/licenses/BSD-3-Clause)
+[![PyTorch](https://img.shields.io/badge/PyTorch-EE4C2C?logo=pytorch&logoColor=white)](https://pytorch.org/)
+
+Author: [Sanyam Bhutani](http://twitter.com/bhutanisanyam1/)
+
+</div>
+
+---
+
+## Why OpenEnv?
+
+Let's take a trip down memory lane:
+
+It's 2016, RL is popular. You read some papers, it looks promising. 
+
+But in real world: Cartpole is the best you can run on a gaming GPU. 
+
+What do you do beyond Cartpole?
+
+Fast-forward to 2025, GRPO is awesome and this time it's not JUST in theory, it works well in practise and is really here!
+
+The problem still remains, how do you take these RL algorithms and take them beyond Cartpole?
+
+A huge part of RL is giving your algorithms environment access to learn. 
+
+We are excited to introduce an Environment Spec for adding Open Environments for RL Training. This will allow you to focus on your experiments and allow everyone to bring their environments.
+
+Focus on experiments, use OpenEnvironments, and build agents that go beyond Cartpole on a single spec.
+
+---
+
+## 📋 What You'll Learn
+
+<table>
+<tr>
+<td width="50%">
+
+**🎯 Part 1-2: The Fundamentals**
+
+- ⚡ RL in 60 seconds
+- 🤔 Why existing solutions fall short
+- 💡 The OpenEnv solution
+
+</td>
+<td width="50%">
+
+**🏗️ Part 3-5: The Architecture**
+
+- 🔧 How OpenEnv works
+- 🔍 Exploring real code
+- 🎮 OpenSpiel integration example
+
+</td>
+</tr>
+<tr>
+<td width="50%">
+
+**🎮 Part 6-8: Hands-On Demo**
+
+- 🔌 Use existing OpenSpiel environment
+- 🤖 Test 4 different policies
+- 👀 Watch learning happen live
+
+</td>
+<td width="50%">
+
+**🔧 Part 9-10: Going Further**
+
+- 🎮 Switch to other OpenSpiel games
+- ✨ Build your own integration
+- 🌐 Deploy to production
+
+</td>
+</tr>
+</table>
+
+!!! tip "Pro Tip"
+    This notebook is designed to run top-to-bottom in Google Colab with zero setup!
+    
+    ⏱️ **Time**: ~5 minutes | 📊 **Difficulty**: Beginner-friendly | 🎯 **Outcome**: Production-ready RL knowledge
+
+---
+
+## 📑 Table of Contents
+
+### Foundation
+
+- [Part 1: RL in 60 Seconds ⏱️](#part-1-rl-in-60-seconds)
+- [Part 2: The Problem with Traditional RL 😤](#part-2-the-problem-with-traditional-rl)
+- [Part 3: Setup 🛠️](#part-3-setup)
+
+### Architecture
+
+- [Part 4: The OpenEnv Pattern 🏗️](#part-4-the-openenv-pattern)
+- [Part 5: Example Integration - OpenSpiel 🎮](#part-5-example-integration---openspiel)
+
+### Hands-On Demo
+
+- [Part 6: Interactive Demo 🎮](#part-6-using-real-openspiel)
+- [Part 7: Four Policies 🤖](#part-7-four-policies)
+- [Part 8: Policy Competition! 🏆](#part-8-policy-competition)
+
+### Advanced
+
+- [Part 9: Using Real OpenSpiel 🎮](#part-9-switching-to-other-games)
+- [Part 10: Create Your Own Integration 🛠️](#part-10-create-your-own-integration)
+
+### Wrap Up
+
+- [Summary: Your Journey 🎓](#summary-your-journey)
+- [Resources 📚](#resources)
+
+---
+
+## Part 1: RL in 60 Seconds ⏱️
+
+**Reinforcement Learning is simpler than you think.**
+
+It's just a loop:
+
+```python
+while not done:
+    observation = environment.observe()
+    action = policy.choose(observation)
+    reward = environment.step(action)
+    policy.learn(reward)
+```
+
+That's it. That's RL.
+
+Let's see it in action:
+
+```python
+import random
+
+print("🎲 " + "="*58 + " 🎲")
+print("   Number Guessing Game - The Simplest RL Example")
+print("🎲 " + "="*58 + " 🎲")
+
+# Environment setup
+target = random.randint(1, 10)
+guesses_left = 3
+
+print(f"\n🎯 I'm thinking of a number between 1 and 10...")
+print(f"💭 You have {guesses_left} guesses. Let's see how random guessing works!\n")
+
+# The RL Loop - Pure random policy (no learning!)
+while guesses_left > 0:
+    # Policy: Random guessing (no learning yet!)
+    guess = random.randint(1, 10)
+    guesses_left -= 1
+    
+    print(f"💭 Guess #{3-guesses_left}: {guess}", end=" → ")
+    
+    # Reward signal (but we're not using it!)
+    if guess == target:
+        print("🎉 Correct! +10 points")
+        break
+    elif abs(guess - target) <= 2:
+        print("🔥 Warm! (close)")
+    else:
+        print("❄️  Cold! (far)")
+else:
+    print(f"\n💔 Out of guesses. The number was {target}.")
+
+print("\n" + "="*62)
+print("💡 This is RL: Observe → Act → Reward → Repeat")
+print("   But this policy is terrible! It doesn't learn from rewards.")
+print("="*62 + "\n")
+```
+
+**Output:**
+```
+🎲 ========================================================== 🎲
+   Number Guessing Game - The Simplest RL Example
+🎲 ========================================================== 🎲
+
+🎯 I'm thinking of a number between 1 and 10...
+💭 You have 3 guesses. Let's see how random guessing works!
+
+💭 Guess #1: 2 → ❄️  Cold! (far)
+💭 Guess #2: 10 → 🎉 Correct! +10 points
+
+==============================================================
+💡 This is RL: Observe → Act → Reward → Repeat
+   But this policy is terrible! It doesn't learn from rewards.
+==============================================================
+```
+
+---
+
+## Part 2: The Problem with Traditional RL 😤
+
+### 🤔 Why Can't We Just Use OpenAI Gym?
+
+Good question! Gym is great for research, but production needs more...
+
+| Challenge | Traditional Approach | OpenEnv Solution |
+|-----------|---------------------|------------------|
+| **Type Safety** | ❌ `obs[0][3]` - what is this? | ✅ `obs.info_state` - IDE knows! |
+| **Isolation** | ❌ Same process (can crash your training) | ✅ Docker containers (fully isolated) |
+| **Deployment** | ❌ "Works on my machine" 🤷 | ✅ Same container everywhere 🐳 |
+| **Scaling** | ❌ Hard to distribute | ✅ Deploy to Kubernetes ☸️ |
+| **Language** | ❌ Python only | ✅ Any language (HTTP API) 🌐 |
+| **Debugging** | ❌ Cryptic numpy errors | ✅ Clear type errors 🐛 |
+
+### 💡 The OpenEnv Philosophy
+
+**"RL environments should be like microservices"**
+
+Think of it like this: You don't run your database in the same process as your web server, right? Same principle!
+
+- 🔒 **Isolated**: Run in containers (security + stability)
+- 🌐 **Standard**: HTTP API, works everywhere
+- 📦 **Versioned**: Docker images (reproducibility!)
+- 🚀 **Scalable**: Deploy to cloud with one command
+- 🛡️ **Type-safe**: Catch bugs before they happen
+- 🔄 **Portable**: Works on Mac, Linux, Windows, Cloud
+
+### The Architecture
+
+```
+┌────────────────────────────────────────────────────────────┐
+│  YOUR TRAINING CODE                                        │
+│                                                            │
+│  env = OpenSpielEnv(...)        ← Import the client      │
+│  result = env.reset()           ← Type-safe!             │
+│  result = env.step(action)      ← Type-safe!             │
+│                                                            │
+└─────────────────┬──────────────────────────────────────────┘
+                  │
+                  │  HTTP/JSON (Language-Agnostic)
+                  │  POST /reset, POST /step, GET /state
+                  │
+┌─────────────────▼──────────────────────────────────────────┐
+│  DOCKER CONTAINER                                          │
+│                                                            │
+│  ┌──────────────────────────────────────────────┐         │
+│  │  FastAPI Server                              │         │
+│  │  └─ Environment (reset, step, state)         │         │
+│  │     └─ Your Game/Simulation Logic            │         │
+│  └──────────────────────────────────────────────┘         │
+│                                                            │
+│  Isolated • Reproducible • Secure                          │
+└────────────────────────────────────────────────────────────┘
+```
+
+!!! info "Key Insight"
+    You never see HTTP details - just clean Python methods!
+
+    ```python
+    env.reset()    # Under the hood: HTTP POST to /reset
+    env.step(...)  # Under the hood: HTTP POST to /step
+    env.state()    # Under the hood: HTTP GET to /state
+    ```
+
+    The magic? OpenEnv handles all the plumbing. You focus on RL! ✨
+
+---
+
+## Part 3: Setup 🛠️
+
+**Running in Colab?** This cell will clone OpenEnv and install dependencies automatically.
+
+**Running locally?** Make sure you're in the OpenEnv directory.
+
+```python
+# Detect environment
+try:
+    import google.colab
+    IN_COLAB = True
+    print("🌐 Running in Google Colab - Perfect!")
+except ImportError:
+    IN_COLAB = False
+    print("💻 Running locally - Nice!")
+
+if IN_COLAB:
+    print("\n📦 Cloning OpenEnv repository...")
+    !git clone https://github.com/meta-pytorch/OpenEnv.git > /dev/null 2>&1
+    %cd OpenEnv
+    
+    print("📚 Installing dependencies (this takes ~10 seconds)...")
+    !pip install -q fastapi uvicorn requests
+    
+    import sys
+    sys.path.insert(0, './src')
+    print("\n✅ Setup complete! Everything is ready to go! 🎉")
+else:
+    import sys
+    from pathlib import Path
+    sys.path.insert(0, str(Path.cwd().parent / 'src'))
+    print("✅ Using local OpenEnv installation")
+
+print("\n🚀 Ready to explore OpenEnv and build amazing things!")
+print("💡 Tip: Run cells top-to-bottom for the best experience.\n")
+```
+
+**Output:**
+```
+💻 Running locally - Nice!
+✅ Using local OpenEnv installation
+
+🚀 Ready to explore OpenEnv and build amazing things!
+💡 Tip: Run cells top-to-bottom for the best experience.
+```
+
+---
+
+## Part 4: The OpenEnv Pattern 🏗️
+
+### Every OpenEnv Environment Has 3 Components:
+
+```
+src/envs/your_env/
+├── 📝 models.py          ← Type-safe contracts
+│                           (Action, Observation, State)
+│
+├── 📱 client.py          ← What YOU import
+│                           (HTTPEnvClient implementation)
+│
+└── 🖥️  server/
+    ├── environment.py    ← Game/simulation logic
+    ├── app.py            ← FastAPI server
+    └── Dockerfile        ← Container definition
+```
+
+Let's explore the actual OpenEnv code to see how this works:
+
+```python
+# Import OpenEnv's core abstractions
+from core.env_server import Environment, Action, Observation, State
+from core.http_env_client import HTTPEnvClient
+
+print("="*70)
+print("   🧩 OPENENV CORE ABSTRACTIONS")
+print("="*70)
+
+print("""
+🖥️  SERVER SIDE (runs in Docker):
+
+    class Environment(ABC):
+        '''Base class for all environment implementations'''
+        
+        @abstractmethod
+        def reset(self) -> Observation:
+            '''Start new episode'''
+        
+        @abstractmethod
+        def step(self, action: Action) -> Observation:
+            '''Execute action, return observation'''
+        
+        @property
+        def state(self) -> State:
+            '''Get episode metadata'''
+
+📱 CLIENT SIDE (your training code):
+
+    class HTTPEnvClient(ABC):
+        '''Base class for HTTP clients'''
+        
+        def reset(self) -> StepResult:
+            # HTTP POST /reset
+        
+        def step(self, action) -> StepResult:
+            # HTTP POST /step
+        
+        def state(self) -> State:
+            # HTTP GET /state
+""")
+
+print("="*70)
+print("\n✨ Same interface on both sides - communication via HTTP!")
+print("🎯 You focus on RL, OpenEnv handles the infrastructure.\n")
+```
+
+**Output:**
+```
+======================================================================
+   🧩 OPENENV CORE ABSTRACTIONS
+======================================================================
+
+🖥️  SERVER SIDE (runs in Docker):
+
+    class Environment(ABC):
+        '''Base class for all environment implementations'''
+        
+        @abstractmethod
+        def reset(self) -> Observation:
+            '''Start new episode'''
+        
+        @abstractmethod
+        def step(self, action: Action) -> Observation:
+            '''Execute action, return observation'''
+        
+        @property
+        def state(self) -> State:
+            '''Get episode metadata'''
+
+📱 CLIENT SIDE (your training code):
+
+    class HTTPEnvClient(ABC):
+        '''Base class for HTTP clients'''
+        
+        def reset(self) -> StepResult:
+            # HTTP POST /reset
+        
+        def step(self, action) -> StepResult:
+            # HTTP POST /step
+        
+        def state(self) -> State:
+            # HTTP GET /state
+
+======================================================================
+
+✨ Same interface on both sides - communication via HTTP!
+🎯 You focus on RL, OpenEnv handles the infrastructure.
+```
+
+---
+
+## Part 5: Example Integration - OpenSpiel 🎮
+
+### What is OpenSpiel?
+
+**OpenSpiel** is a library from DeepMind with **70+ game environments** for RL research.
+
+### OpenEnv's Integration
+
+We've wrapped **6 OpenSpiel games** following the OpenEnv pattern:
+
+| **🎯 Single-Player** | **👥 Multi-Player** |
+|---------------------|---------------------|
+| 1. **Catch** - Catch falling ball | 5. **Tic-Tac-Toe** - Classic 3×3 |
+| 2. **Cliff Walking** - Navigate grid | 6. **Kuhn Poker** - Imperfect info poker |
+| 3. **2048** - Tile puzzle | |
+| 4. **Blackjack** - Card game | |
+
+This shows how OpenEnv can wrap **any** existing RL library!
+
+```python
+from envs.openspiel_env.client import OpenSpielEnv
+
+print("="*70)
+print("   🔌 HOW OPENENV WRAPS OPENSPIEL")
+print("="*70)
+
+print("""
+class OpenSpielEnv(HTTPEnvClient[OpenSpielAction, OpenSpielObservation]):
+    
+    def _step_payload(self, action: OpenSpielAction) -> dict:
+        '''Convert typed action to JSON for HTTP'''
+        return {
+            "action_id": action.action_id,
+            "game_name": action.game_name,
+        }
+    
+    def _parse_result(self, payload: dict) -> StepResult:
+        '''Parse HTTP JSON response into typed observation'''
+        return StepResult(
+            observation=OpenSpielObservation(...),
+            reward=payload['reward'],
+            done=payload['done']
+        )
+
+""")
+
+print("─" * 70)
+print("\n✨ Usage (works for ALL OpenEnv environments):")
+print("""
+  env = OpenSpielEnv(base_url="http://localhost:8000")
+  
+  result = env.reset()
+  # Returns StepResult[OpenSpielObservation] - Type safe!
+  
+  result = env.step(OpenSpielAction(action_id=2, game_name="catch"))
+  # Type checker knows this is valid!
+  
+  state = env.state()
+  # Returns OpenSpielState
+""")
+
+print("─" * 70)
+print("\n🎯 This pattern works for ANY environment you want to wrap!\n")
+```
+
+**Output:**
+```
+======================================================================
+   🔌 HOW OPENENV WRAPS OPENSPIEL
+======================================================================
+
+class OpenSpielEnv(HTTPEnvClient[OpenSpielAction, OpenSpielObservation]):
+    
+    def _step_payload(self, action: OpenSpielAction) -> dict:
+        '''Convert typed action to JSON for HTTP'''
+        return {
+            "action_id": action.action_id,
+            "game_name": action.game_name,
+        }
+    
+    def _parse_result(self, payload: dict) -> StepResult:
+        '''Parse HTTP JSON response into typed observation'''
+        return StepResult(
+            observation=OpenSpielObservation(...),
+            reward=payload['reward'],
+            done=payload['done']
+        )
+
+
+──────────────────────────────────────────────────────────────────────
+
+✨ Usage (works for ALL OpenEnv environments):
+
+  env = OpenSpielEnv(base_url="http://localhost:8000")
+  
+  result = env.reset()
+  # Returns StepResult[OpenSpielObservation] - Type safe!
+  
+  result = env.step(OpenSpielAction(action_id=2, game_name="catch"))
+  # Type checker knows this is valid!
+  
+  state = env.state()
+  # Returns OpenSpielState
+
+──────────────────────────────────────────────────────────────────────
+
+🎯 This pattern works for ANY environment you want to wrap!
+```
+
+### Type-Safe Models
+
+```python
+# Import OpenSpiel integration models
+from envs.openspiel_env.models import (
+    OpenSpielAction,
+    OpenSpielObservation,
+    OpenSpielState
+)
+from dataclasses import fields
+
+print("="*70)
+print("   🎮 OPENSPIEL INTEGRATION - TYPE-SAFE MODELS")
+print("="*70)
+
+print("\n📤 OpenSpielAction (what you send):")
+print("   " + "─" * 64)
+for field in fields(OpenSpielAction):
+    print(f"   • {field.name:20s} : {field.type}")
+
+print("\n📥 OpenSpielObservation (what you receive):")
+print("   " + "─" * 64)
+for field in fields(OpenSpielObservation):
+    print(f"   • {field.name:20s} : {field.type}")
+
+print("\n📊 OpenSpielState (episode metadata):")
+print("   " + "─" * 64)
+for field in fields(OpenSpielState):
+    print(f"   • {field.name:20s} : {field.type}")
+
+print("\n" + "="*70)
+print("\n💡 Type safety means:")
+print("   ✅ Your IDE autocompletes these fields")
+print("   ✅ Typos are caught before running")
+print("   ✅ Refactoring is safe")
+print("   ✅ Self-documenting code\n")
+```
+
+**Output:**
+```
+======================================================================
+   🎮 OPENSPIEL INTEGRATION - TYPE-SAFE MODELS
+======================================================================
+
+📤 OpenSpielAction (what you send):
+   ────────────────────────────────────────────────────────────────
+   • metadata             : typing.Dict[str, typing.Any]
+   • action_id            : int
+   • game_name            : str
+   • game_params          : Dict[str, Any]
+
+📥 OpenSpielObservation (what you receive):
+   ────────────────────────────────────────────────────────────────
+   • done                 : <class 'bool'>
+   • reward               : typing.Union[bool, int, float, NoneType]
+   • metadata             : typing.Dict[str, typing.Any]
+   • info_state           : List[float]
+   • legal_actions        : List[int]
+   • game_phase           : str
+   • current_player_id    : int
+   • opponent_last_action : Optional[int]
+
+📊 OpenSpielState (episode metadata):
+   ────────────────────────────��───────────────────────────────────
+   • episode_id           : typing.Optional[str]
+   • step_count           : <class 'int'>
+   • game_name            : str
+   • agent_player         : int
+   • opponent_policy      : str
+   • game_params          : Dict[str, Any]
+   • num_players          : int
+
+======================================================================
+
+💡 Type safety means:
+   ✅ Your IDE autocompletes these fields
+   ✅ Typos are caught before running
+   ✅ Refactoring is safe
+   ✅ Self-documenting code
+```
+
+### How the Client Works
+
+The client **inherits from HTTPEnvClient** and implements 3 methods:
+
+1. `_step_payload()` - Convert action → JSON
+2. `_parse_result()` - Parse JSON → typed observation  
+3. `_parse_state()` - Parse JSON → state
+
+That's it! The base class handles all HTTP communication.
+
+---
+
+## Part 6: Using Real OpenSpiel 🎮
+
+<div style="text-align: center; background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); color: white; padding: 30px; border-radius: 15px; margin: 30px 0;">
+
+### Now let's USE a production environment!
+
+We'll play **Catch** using OpenEnv's **OpenSpiel integration** 🎯
+
+This is a REAL environment running in production at companies!
+
+**Get ready for:**
+
+- 🔌 Using existing environments (not building)
+- 🤖 Testing policies against real games
+- 📊 Live gameplay visualization
+- 🎯 Production-ready patterns
+
+</div>
+
+### The Game: Catch 🔴🏓
+
+```
+⬜ ⬜ 🔴 ⬜ ⬜
+⬜ ⬜ ⬜ ⬜ ⬜
+⬜ ⬜ ⬜ ⬜ ⬜   Ball
+⬜ ⬜ ⬜ ⬜ ⬜
+⬜ ⬜ ⬜ ⬜ ⬜   falls
+⬜ ⬜ ⬜ ⬜ ⬜
+⬜ ⬜ ⬜ ⬜ ⬜   down
+⬜ ⬜ ⬜ ⬜ ⬜
+⬜ ⬜ ⬜ ⬜ ⬜
+⬜ ⬜ 🏓 ⬜ ⬜
+     Paddle
+```
+
+**Rules:**
+
+- 10×5 grid
+- Ball falls from random column
+- Move paddle left/right to catch it
+
+**Actions:**
+
+- `0` = Move LEFT ⬅️
+- `1` = STAY 🛑
+- `2` = Move RIGHT ➡️
+
+**Reward:**
+
+- `+1` if caught 🎉
+- `0` if missed 😢
+
+!!! note "Why Catch?"
+    - Simple rules (easy to understand)
+    - Fast episodes (~5 steps)
+    - Clear success/failure
+    - Part of OpenSpiel's 70+ games!
+
+    **💡 The Big Idea:**
+    Instead of building this from scratch, we'll USE OpenEnv's existing OpenSpiel integration. Same interface, but production-ready!
+
+```python
+from envs.openspiel_env import OpenSpielEnv
+from envs.openspiel_env.models import (
+    OpenSpielAction,
+    OpenSpielObservation,
+    OpenSpielState
+)
+from dataclasses import fields
+
+print("🎮 " + "="*64 + " 🎮")
+print("   ✅ Importing Real OpenSpiel Environment!")
+print("🎮 " + "="*64 + " 🎮\n")
+
+print("📦 What we just imported:")
+print("   • OpenSpielEnv - HTTP client for OpenSpiel games")
+print("   • OpenSpielAction - Type-safe actions")
+print("   • OpenSpielObservation - Type-safe observations")
+print("   • OpenSpielState - Episode metadata\n")
+
+print("📋 OpenSpielObservation fields:")
+print("   " + "─" * 60)
+for field in fields(OpenSpielObservation):
+    print(f"   • {field.name:25s} : {field.type}")
+
+print("\n" + "="*70)
+print("\n💡 This is REAL OpenEnv code - used in production!")
+print("   • Wraps 6 OpenSpiel games (Catch, Tic-Tac-Toe, Poker, etc.)")
+print("   • Type-safe actions and observations")
+print("   • Works via HTTP (we'll see that next!)\n")
+```
+
+**Output:**
+```
+🎮 ================================================================ 🎮
+   ✅ Importing Real OpenSpiel Environment!
+🎮 ================================================================ 🎮
+
+📦 What we just imported:
+   • OpenSpielEnv - HTTP client for OpenSpiel games
+   • OpenSpielAction - Type-safe actions
+   • OpenSpielObservation - Type-safe observations
+   • OpenSpielState - Episode metadata
+
+📋 OpenSpielObservation fields:
+   ────────────────────────────────────────────────────────────
+   • done                      : <class 'bool'>
+   • reward                    : typing.Union[bool, int, float, NoneType]
+   • metadata                  : typing.Dict[str, typing.Any]
+   • info_state                : List[float]
+   • legal_actions             : List[int]
+   • game_phase                : str
+   • current_player_id         : int
+   • opponent_last_action      : Optional[int]
+
+======================================================================
+
+💡 This is REAL OpenEnv code - used in production!
+   • Wraps 6 OpenSpiel games (Catch, Tic-Tac-Toe, Poker, etc.)
+   • Type-safe actions and observations
+   • Works via HTTP (we'll see that next!)
+```
+
+---
+
+## Part 7: Four Policies 🤖
+
+Let's test 4 different AI strategies:
+
+| Policy | Strategy | Expected Performance |
+|--------|----------|----------------------|
+| **🎲 Random** | Pick random action every step | ~20% (pure luck) |
+| **🛑 Always Stay** | Never move, hope ball lands in center | ~20% (terrible!) |
+| **🧠 Smart** | Move paddle toward ball | 100% (optimal!) |
+| **📈 Learning** | Start random, learn smart strategy | ~85% (improves over time) |
+
+**💡 These policies work with ANY OpenSpiel game!**
+
+```python
+import random
+
+# ============================================================================
+# POLICIES - Different AI strategies (adapted for OpenSpiel)
+# ============================================================================
+
+class RandomPolicy:
+    """Baseline: Pure random guessing."""
+    name = "🎲 Random Guesser"
+
+    def select_action(self, obs: OpenSpielObservation) -> int:
+        return random.choice(obs.legal_actions)
+
+
+class AlwaysStayPolicy:
+    """Bad strategy: Never moves."""
+    name = "🛑 Always Stay"
+
+    def select_action(self, obs: OpenSpielObservation) -> int:
+        return 1  # STAY
+
+
+class SmartPolicy:
+    """Optimal: Move paddle toward ball."""
+    name = "🧠 Smart Heuristic"
+
+    def select_action(self, obs: OpenSpielObservation) -> int:
+        # Parse OpenSpiel observation
+        # For Catch: info_state is a flattened 10x5 grid
+        # Ball position and paddle position encoded in the vector
+        info_state = obs.info_state
+
+        # Find ball and paddle positions from info_state
+        # Catch uses a 10x5 grid, so 50 values
+        grid_size = 5
+
+        # Find positions (ball = 1.0 in the flattened grid, paddle = 1.0 in the last row of the flattened grid)
+        ball_col = None
+        paddle_col = None
+
+        for idx, val in enumerate(info_state):
+            if abs(val - 1.0) < 0.01:  # Ball
+                ball_col = idx % grid_size
+                break
+
+        last_row = info_state[-grid_size:]
+        paddle_col = last_row.index(1.0) # Paddle
+
+        if ball_col is not None and paddle_col is not None:
+            if paddle_col < ball_col:
+                return 2  # Move RIGHT
+            elif paddle_col > ball_col:
+                return 0  # Move LEFT
+
+        return 1  # STAY (fallback)
+
+
+class LearningPolicy:
+    """Simulated RL: Epsilon-greedy exploration."""
+    name = "📈 Learning Agent"
+
+    def __init__(self):
+        self.steps = 0
+        self.smart_policy = SmartPolicy()
+
+    def select_action(self, obs: OpenSpielObservation) -> int:
+        self.steps += 1
+
+        # Decay exploration rate over time
+        epsilon = max(0.1, 1.0 - (self.steps / 100))
+
+        if random.random() < epsilon:
+            # Explore: random action
+            return random.choice(obs.legal_actions)
+        else:
+            # Exploit: use smart strategy
+            return self.smart_policy.select_action(obs)
+
+
+print("🤖 " + "="*64 + " 🤖")
+print("   ✅ 4 Policies Created (Adapted for OpenSpiel)!")
+print("🤖 " + "="*64 + " 🤖\n")
+
+policies = [RandomPolicy(), AlwaysStayPolicy(), SmartPolicy(), LearningPolicy()]
+for i, policy in enumerate(policies, 1):
+    print(f"   {i}. {policy.name}")
+
+print("\n💡 These policies work with OpenSpielObservation!")
+print("   • Read info_state (flattened grid)")
+print("   • Use legal_actions")
+print("   • Work with ANY OpenSpiel game that exposes these!\n")
+```
+
+**Output:**
+```
+🤖 ================================================================ 🤖
+   ✅ 4 Policies Created (Adapted for OpenSpiel)!
+🤖 ================================================================ 🤖
+
+   1. 🎲 Random Guesser
+   2. 🛑 Always Stay
+   3. 🧠 Smart Heuristic
+   4. 📈 Learning Agent
+
+💡 These policies work with OpenSpielObservation!
+   • Read info_state (flattened grid)
+   • Use legal_actions
+   • Work with ANY OpenSpiel game that exposes these!
+```
+
+---
+
+## Part 8: Policy Competition! 🏆
+
+Let's run **50 episodes** for each policy against **REAL OpenSpiel** and see who wins!
+
+This is production code - every action is an HTTP call to the OpenSpiel server!
+
+```python
+def evaluate_policies(env, num_episodes=50):
+    """Compare all policies over many episodes using real OpenSpiel."""
+    policies = [
+        RandomPolicy(),
+        AlwaysStayPolicy(),
+        SmartPolicy(),
+        LearningPolicy(),
+    ]
+
+    print("\n🏆 " + "="*66 + " 🏆")
+    print(f"   POLICY SHOWDOWN - {num_episodes} Episodes Each")
+    print(f"   Playing against REAL OpenSpiel Catch!")
+    print("🏆 " + "="*66 + " 🏆\n")
+
+    results = []
+    for policy in policies:
+        print(f"⚡ Testing {policy.name}...", end=" ")
+        successes = sum(run_episode(env, policy, visualize=False)
+                       for _ in range(num_episodes))
+        success_rate = (successes / num_episodes) * 100
+        results.append((policy.name, success_rate, successes))
+        print(f"✓ Done!")
+
+    print("\n" + "="*70)
+    print("   📊 FINAL RESULTS")
+    print("="*70 + "\n")
+
+    # Sort by success rate (descending)
+    results.sort(key=lambda x: x[1], reverse=True)
+
+    # Award medals to top 3
+    medals = ["🥇", "🥈", "🥉", "  "]
+
+    for i, (name, rate, successes) in enumerate(results):
+        medal = medals[i]
+        bar = "█" * int(rate / 2)
+        print(f"{medal} {name:25s} [{bar:<50}] {rate:5.1f}% ({successes}/{num_episodes})")
+
+    print("\n" + "="*70)
+    print("\n✨ Key Insights:")
+    print("   • Random (~20%):      Baseline - pure luck 🎲")
+    print("   • Always Stay (~20%): Bad strategy - stays center 🛑")
+    print("   • Smart (100%):       Optimal - perfect play! 🧠")
+    print("   • Learning (~85%):    Improves over time 📈")
+    print("\n🎓 This is Reinforcement Learning + OpenEnv in action:")
+    print("   1. We USED existing OpenSpiel environment (didn't build it)")
+    print("   2. Type-safe communication over HTTP")
+    print("   3. Same code works for ANY OpenSpiel game")
+    print("   4. Production-ready architecture\n")
+
+# Run the epic competition!
+print("🎮 Starting the showdown against REAL OpenSpiel...\n")
+evaluate_policies(client, num_episodes=50)
+```
+
+---
+
+## Part 9: Switching to Other Games 🎮
+
+### What We Just Used: Real OpenSpiel! 🎉
+
+In Parts 6-8, we **USED** the existing OpenSpiel Catch environment:
+
+| What We Did | How It Works |
+|-------------|--------------|
+| **Imported** | OpenSpielEnv client (pre-built) |
+| **Started** | OpenSpiel server via uvicorn |
+| **Connected** | HTTP client to server |
+| **Played** | Real OpenSpiel Catch game |
+
+**🎯 This is production code!** Every action was an HTTP call to a real OpenSpiel environment.
+
+### 🎮 6 Games Available - Same Interface!
+
+The beauty of OpenEnv? **Same code, different games!**
+
+```python
+# We just used Catch
+env = OpenSpielEnv(base_url="http://localhost:8000")
+# game_name="catch" was set via environment variable
+
+# Want Tic-Tac-Toe instead? Just change the game!
+# Start server with: OPENSPIEL_GAME=tic_tac_toe uvicorn ...
+# Same client code works!
+```
+
+**🎮 All 6 Games:**
+
+1. ✅ **`catch`** - What we just used!
+2. **`tic_tac_toe`** - Classic 3×3
+3. **`kuhn_poker`** - Imperfect information poker
+4. **`cliff_walking`** - Grid navigation
+5. **`2048`** - Tile puzzle
+6. **`blackjack`** - Card game
+
+**All use the exact same OpenSpielEnv client!**
+
+### Try Another Game (Optional):
+
+```python
+# Stop the current server (kill the server_process)
+# Then start a new game:
+
+server_process = subprocess.Popen(
+    [sys.executable, "-m", "uvicorn",
+     "envs.openspiel_env.server.app:app",
+     "--host", "0.0.0.0",
+     "--port", "8000"],
+    env={**os.environ,
+         "PYTHONPATH": f"{work_dir}/src",
+         "OPENSPIEL_GAME": "tic_tac_toe",  # Changed!
+         "OPENSPIEL_AGENT_PLAYER": "0",
+         "OPENSPIEL_OPPONENT_POLICY": "random"},
+    # ... rest of config
+)
+
+# Same client works!
+client = OpenSpielEnv(base_url="http://localhost:8000")
+result = client.reset()  # Now playing Tic-Tac-Toe!
+```
+
+**💡 Key Insight**: You don't rebuild anything - you just USE different games with the same client!
+
+---
+
+## Part 10: Create Your Own Integration 🛠️
+
+### The 5-Step Pattern
+
+Want to wrap your own environment in OpenEnv? Here's how:
+
+### Step 1: Define Types (`models.py`)
+
+```python
+from dataclasses import dataclass
+from core.env_server import Action, Observation, State
+
+@dataclass
+class YourAction(Action):
+    action_value: int
+    # Add your action fields
+
+@dataclass
+class YourObservation(Observation):
+    state_data: List[float]
+    done: bool
+    reward: float
+    # Add your observation fields
+
+@dataclass
+class YourState(State):
+    episode_id: str
+    step_count: int
+    # Add your state fields
+```
+
+### Step 2: Implement Environment (`server/environment.py`)
+
+```python
+from core.env_server import Environment
+
+class YourEnvironment(Environment):
+    def reset(self) -> Observation:
+        # Initialize your game/simulation
+        return YourObservation(...)
+    
+    def step(self, action: Action) -> Observation:
+        # Execute action, update state
+        return YourObservation(...)
+    
+    @property
+    def state(self) -> State:
+        return self._state
+```
+
+### Step 3: Create Client (`client.py`)
+
+```python
+from core.http_env_client import HTTPEnvClient
+from core.types import StepResult
+
+class YourEnv(HTTPEnvClient[YourAction, YourObservation]):
+    def _step_payload(self, action: YourAction) -> dict:
+        """Convert action to JSON"""
+        return {"action_value": action.action_value}
+    
+    def _parse_result(self, payload: dict) -> StepResult:
+        """Parse JSON to observation"""
+        return StepResult(
+            observation=YourObservation(...),
+            reward=payload['reward'],
+            done=payload['done']
+        )
+    
+    def _parse_state(self, payload: dict) -> YourState:
+        return YourState(...)
+```
+
+### Step 4: Create Server (`server/app.py`)
+
+```python
+from core.env_server import create_fastapi_app
+from .your_environment import YourEnvironment
+
+env = YourEnvironment()
+app = create_fastapi_app(env)
+
+# That's it! OpenEnv creates all endpoints for you.
+```
+
+### Step 5: Dockerize (`server/Dockerfile`)
+
+```dockerfile
+FROM python:3.11-slim
+
+WORKDIR /app
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . .
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000"]
+```
+
+### 🎓 Examples to Study
+
+OpenEnv includes 3 complete examples:
+
+1. **`src/envs/echo_env/`**
+   - Simplest possible environment
+   - Great for testing and learning
+
+2. **`src/envs/openspiel_env/`**
+   - Wraps external library (OpenSpiel)
+   - Shows integration pattern
+   - 6 games in one integration
+
+3. **`src/envs/coding_env/`**
+   - Python code execution environment
+   - Shows complex use case
+   - Security considerations
+
+**💡 Study these to understand the patterns!**
+
+---
+
+## 🎓 Summary: Your Journey
+
+### What You Learned
+
+<table>
+<tr>
+<td width="50%" style="vertical-align: top;">
+
+### 📚 Concepts
+
+✅ **RL Fundamentals**
+
+- The observe-act-reward loop
+- What makes good policies
+- Exploration vs exploitation
+
+✅ **OpenEnv Architecture**
+
+- Client-server separation
+- Type-safe contracts
+- HTTP communication layer
+
+✅ **Production Patterns**
+
+- Docker isolation
+- API design
+- Reproducible deployments
+
+</td>
+<td width="50%" style="vertical-align: top;">
+
+### 🛠️ Skills
+
+✅ **Using Environments**
+
+- Import OpenEnv clients
+- Call reset/step/state
+- Work with typed observations
+
+✅ **Building Environments**
+
+- Define type-safe models
+- Implement Environment class
+- Create HTTPEnvClient
+
+✅ **Testing & Debugging**
+
+- Compare policies
+- Visualize episodes
+- Measure performance
+
+</td>
+</tr>
+</table>
+
+### OpenEnv vs Traditional RL
+
+| Feature | Traditional (Gym) | OpenEnv | Winner |
+|---------|------------------|---------|--------|
+| **Type Safety** | ❌ Arrays, dicts | ✅ Dataclasses | 🏆 OpenEnv |
+| **Isolation** | ❌ Same process | ✅ Docker | 🏆 OpenEnv |
+| **Deployment** | ❌ Manual setup | ✅ K8s-ready | 🏆 OpenEnv |
+| **Language** | ❌ Python only | ✅ Any (HTTP) | 🏆 OpenEnv |
+| **Reproducibility** | ❌ "Works on my machine" | ✅ Same everywhere | 🏆 OpenEnv |
+| **Community** | ✅ Large ecosystem | 🟡 Growing | 🤝 Both! |
+
+!!! success "The Bottom Line"
+    OpenEnv brings **production engineering** to RL:
+    
+    - Same environments work locally and in production
+    - Type safety catches bugs early
+    - Docker isolation prevents conflicts
+    - HTTP API works with any language
+
+    **It's RL for 2024 and beyond.**
+
+---
+
+## 📚 Resources
+
+### 🔗 Essential Links
+
+- **🏠 OpenEnv GitHub**: https://github.com/meta-pytorch/OpenEnv
+- **🎮 OpenSpiel**: https://github.com/google-deepmind/open_spiel
+- **⚡ FastAPI Docs**: https://fastapi.tiangolo.com/
+- **🐳 Docker Guide**: https://docs.docker.com/get-started/
+- **🔥 PyTorch**: https://pytorch.org/
+
+### 📖 Documentation Deep Dives
+
+- **Environment Creation Guide**: `src/envs/README.md`
+- **OpenSpiel Integration**: `src/envs/openspiel_env/README.md`
+- **Example Scripts**: `examples/`
+- **RFC 001**: [Baseline API Specs](https://github.com/meta-pytorch/OpenEnv/pull/26)
+
+### 🎓 Community & Support
+
+**Supported by amazing organizations:**
+
+- 🔥 Meta PyTorch
+- 🤗 Hugging Face
+- ⚡ Unsloth AI
+- 🌟 Reflection AI
+- 🚀 And many more!
+
+**License**: BSD 3-Clause (very permissive!)
+
+**Contributions**: Always welcome! Check out the issues tab.
+
+---
+
+### 🌈 What's Next?
+
+1. ⭐ **Star the repo** to show support and stay updated
+2. 🔄 **Try modifying** the Catch game (make it harder? bigger grid?)
+3. 🎮 **Explore** other OpenSpiel games
+4. 🛠️ **Build** your own environment integration
+5. 💬 **Share** what you build with the community!
+

tutorial/02-deployment.md

@@ -0,0 +1,427 @@
+# 2. Deploying an OpenEnv environment
+
+This section covers deploying OpenEnv environments locally, on clusters, and on Hugging Face Spaces.
+
+**Contents:**
+- [Local Development with Uvicorn](#local-development-with-uvicorn)
+- [Docker Deployment](#docker-deployment)
+- [Hugging Face Spaces](#hugging-face-spaces)
+- [Best Practices](#best-practices)
+
+## HF Spaces are the infrastructure for OpenEnv environments
+
+Every HF Space provides three things that OpenEnv environments need:
+
+| Component | What it provides | How to access | Used as |
+|-----------|------------------|---------------|-----------|
+| **Server** | Running environment endpoint | `https://<username>-<space-name>.hf.space` | Agent and Public API |
+| **Repository** | Installable Python package | `pip install git+https://huggingface.co/spaces/<username>-<space-name>` | Code and client |
+| **Registry** | Docker container image | `docker pull registry.hf.space/<username>-<space-name>:latest` | Deployment |
+
+This means a single Space deployment gives you all the components you need to use an environment in training.
+
+### 1. Server: A running environment endpoint
+
+When you deploy to HF Spaces, your environment runs as a server. The client connects via **WebSocket** (`/ws`) for a persistent session:
+
+```python
+from echo_env import EchoEnv, EchoAction
+
+# Connect directly to the running Space (WebSocket under the hood)
+# Async (recommended):
+async with EchoEnv(base_url="https://openenv-echo-env.hf.space") as client:
+    result = await client.reset()
+    result = await client.step(EchoAction(message="Hello"))
+
+# Sync (using .sync() wrapper):
+with EchoEnv(base_url="https://openenv-echo-env.hf.space").sync() as client:
+    result = client.reset()
+    result = client.step(EchoAction(message="Hello"))
+```
+
+**Endpoints available:**
+
+| Endpoint | Protocol | Description |
+|----------|----------|-------------|
+| `/ws` | **WebSocket** | Persistent session (used by client) |
+| `/health` | HTTP GET | Health check |
+| `/reset` | HTTP POST | Reset environment (stateless) |
+| `/step` | HTTP POST | Execute action (stateless) |
+| `/state` | HTTP GET | Get current state |
+| `/docs` | HTTP GET | OpenAPI documentation |
+| `/web` | HTTP GET | Interactive web UI |
+
+> **Note:** The Python client uses the `/ws` WebSocket endpoint by default. HTTP endpoints are available for debugging or stateless use cases.
+
+**Example: Check if a Space is running**
+
+```bash
+curl https://openenv-echo-env.hf.space/health
+# {"status": "healthy"}
+```
+
+### 2. Repository: Installable Python package
+
+Every Space is a Git repository. OpenEnv environments include a `pyproject.toml`, making them pip-installable directly from the Space URL.
+
+```bash
+# Install client package from Space
+pip install git+https://huggingface.co/spaces/openenv/echo-env
+```
+
+This installs:
+- **Client class** (`EchoEnv`) — Handles HTTP/WebSocket communication
+- **Models** (`EchoAction`, `EchoObservation`) — Typed action and observation classes
+- **Utilities** — Any helper functions the environment provides
+
+**After installation:**
+
+```python
+from envs.echo_env import EchoEnv, EchoAction, EchoObservation
+
+# Now you have typed classes for the environment
+action = EchoAction(message="Hello")
+```
+
+### 3. Registry: Docker container image
+
+Every Docker-based Space has a container registry. You can pull and run the environment locally.
+
+```bash
+# Pull the image
+docker pull registry.hf.space/openenv-echo-env:latest
+
+# Run locally on port 8001
+docker run -d -p 8001:8000 registry.hf.space/openenv-echo-env:latest
+```
+
+**Find the registry URL for any Space:**
+
+1. Go to the Space page (e.g., [openenv/echo-env](https://huggingface.co/spaces/openenv/echo-env))
+2. Click **⋮** (three dots) → **"Run locally"**
+3. Copy the `docker run` command
+
+### Choosing an access method
+
+| Method | Use when | Pros | Cons |
+|--------|----------|------|------|
+| **Server** | Quick testing, low volume | Zero setup | Network latency, rate limits |
+| **Repository** | Need typed classes | Type safety, IDE support | Still need a server |
+| **Docker** | Local dev, high throughput | Full control, no network | Requires Docker |
+
+**Typical workflow:**
+
+```python
+import asyncio
+from echo_env import EchoEnv, EchoAction
+
+async def main():
+    # Development: connect to remote Space
+    async with EchoEnv(base_url="https://openenv-echo-env.hf.space") as client:
+        result = await client.reset()
+
+    # Production: run locally for speed
+    # docker run -d -p 8001:8000 registry.hf.space/openenv-echo-env:latest
+    async with EchoEnv(base_url="http://localhost:8001") as client:
+        result = await client.reset()
+
+    # Or let the client manage Docker for you
+    client = await EchoEnv.from_env("openenv/echo-env")  # Auto-pulls and runs
+    async with client:
+        result = await client.reset()
+
+asyncio.run(main())
+
+# For sync usage, use the .sync() wrapper:
+with EchoEnv(base_url="http://localhost:8001").sync() as client:
+    result = client.reset()
+```
+
+> **Reference:** [HF Spaces Documentation](https://huggingface.co/docs/hub/spaces) | [Environment Hub Collection](https://huggingface.co/collections/openenv/environment-hub)
+
+
+## Local Development with Uvicorn
+
+The fastest way to iterate on environment logic is running directly with Uvicorn.
+
+## Clone and run the environment locally
+
+```bash
+# Clone from HF Space
+git clone https://huggingface.co/spaces/burtenshaw/openenv-benchmark
+cd openenv-benchmark
+
+# Install in editable mode
+uv sync
+
+# Start server
+uv run server
+
+# Run isolated from remote Space
+uv run --isolated --project https://huggingface.co/spaces/burtenshaw/openenv-benchmark server
+```
+
+## Uvicorn directly in python
+
+```bash
+# Full control over uvicorn options
+uvicorn benchmark.server.app:app --host "$HOST" --port "$PORT" --workers "$WORKERS"
+
+# With reload for development
+uvicorn benchmark.server.app:app --host 0.0.0.0 --port 8000 --reload
+
+# Multi-Worker Mode For better concurrency:
+uvicorn benchmark.server.app:app --host 0.0.0.0 --port 8000 --workers 4
+```
+
+| Flag | Purpose |
+|------|---------|
+| `--reload` | Auto-restart on code changes |
+| `--workers N` | Run N worker processes |
+| `--log-level debug` | Verbose logging |
+
+## Docker Deployment
+
+Docker provides isolation and reproducibility for production use.
+
+### Run the environment locally from the space
+
+```bash
+# Run the environment locally from the space
+docker run -d -p 8000:8000 registry.hf.space/openenv-echo-env:latest
+```
+
+### Build Image
+
+```bash
+# Clone from HF Space
+git clone https://huggingface.co/spaces/burtenshaw/openenv-benchmark
+cd openenv-benchmark
+
+# Using OpenEnv CLI (recommended)
+openenv build -t openenv-benchmark:latest
+
+# Or with Docker directly
+docker build -t openenv-benchmark:latest -f server/Dockerfile .
+```
+
+### Run Container
+
+```bash
+# Basic run
+docker run -d -p 8000:8000 my-env:latest
+
+# With environment variables
+docker run -d -p 8000:8000 \
+    -e WORKERS=4 \
+    -e MAX_CONCURRENT_ENVS=100 \
+    my-env:latest
+
+# Named container for easy management
+docker run -d --name my-env -p 8000:8000 my-env:latest
+```
+
+### Connect from Python
+
+```python
+import asyncio
+from echo_env import EchoEnv, EchoAction
+
+async def main():
+    # Async usage (recommended)
+    async with EchoEnv(base_url="http://localhost:8000") as client:
+        result = await client.reset()
+        result = await client.step(EchoAction(message="Hello"))
+        print(result.observation)
+
+    # From Docker image
+    client = await EchoEnv.from_docker_image("<local_docker_image>")
+    async with client:
+        result = await client.reset()
+        print(result.observation)
+
+asyncio.run(main())
+
+# Sync usage (using .sync() wrapper)
+with EchoEnv(base_url="http://localhost:8000").sync() as client:
+    result = client.reset()
+    result = client.step(EchoAction(message="Hello"))
+    print(result.observation)
+```
+
+### Container Lifecycle
+
+| Method | Container | WebSocket | On `close()` |
+|--------|-----------|-----------|--------------|
+| `from_hub(repo_id)` | Starts | Connects | Stops container |
+| `from_hub(repo_id, use_docker=False)` | None (UV) | Connects | Stops UV server |
+| `from_docker_image(image)` | Starts | Connects | Stops container |
+| `MyEnv(base_url=...)` | None | Connects | Disconnects only |
+
+Find Docker Commands for Any Space
+
+1. Open the Space on HuggingFace Hub
+2. Click **⋮ (three dots)** menu
+3. Select **"Run locally"**
+4. Copy the provided `docker run` command
+
+## Deploy with CLI
+
+```bash
+cd my_env
+
+# Deploy to your namespace
+openenv push
+
+# Deploy to specific repo
+openenv push --repo-id username/my-env
+
+# Deploy as private
+openenv push --repo-id username/my-env --private
+```
+
+### Space Configuration
+
+The `openenv.yaml` manifest controls Space settings:
+
+```yaml
+# openenv.yaml
+name: my_env
+version: "1.0.0"
+description: My custom environment
+```
+
+Hardware Options:
+
+| Tier | vCPU | RAM | Cost |
+|------|------|-----|------|
+| CPU Basic (Free) | 2 | 16GB | Free |
+| CPU Upgrade | 8 | 32GB | $0.03/hr |
+
+OpenEnv environments support configuration via environment variables.
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `WORKERS` | 4 | Uvicorn worker processes |
+| `PORT` | 8000 | Server port |
+| `HOST` | 0.0.0.0 | Bind address |
+| `MAX_CONCURRENT_ENVS` | 100 | Max WebSocket sessions |
+| `ENABLE_WEB_INTERFACE` | Auto | Enable web UI |
+
+### Environment-Specific Variables
+
+Some environments have custom variables:
+
+**TextArena:**
+```bash
+TEXTARENA_ENV_ID=Wordle-v0
+TEXTARENA_NUM_PLAYERS=1
+TEXTARENA_MAX_TURNS=6
+```
+
+**Coding Environment:**
+```bash
+SANDBOX_TIMEOUT=30
+MAX_OUTPUT_LENGTH=10000
+```
+
+# DEMO: Deploying to Hugging Face Spaces
+
+This demo walks through the full workflow: create an environment, test locally, deploy to HF Spaces, and use it.
+
+## Step 1: Initialize a new environment
+
+```bash
+openenv init my_env
+cd my_env
+```
+
+This creates the standard OpenEnv structure:
+
+```
+my_env/
+├── server/
+│   ├── app.py           # FastAPI server
+│   ├── environment.py   # Your environment logic
+│   └── Dockerfile
+├── models.py            # Action/Observation types
+├── client.py            # HTTP client
+├── openenv.yaml         # Manifest
+└── pyproject.toml
+```
+
+## Step 2: Run locally
+
+```bash
+# Start the server
+uv run server
+
+# Or with uvicorn directly
+uvicorn server.app:app --host 0.0.0.0 --port 8000 --reload
+```
+
+Test the health endpoint:
+
+```bash
+curl http://localhost:8000/health
+# {"status": "healthy"}
+```
+
+## Step 3: Deploy to HF Spaces
+
+```bash
+openenv push --repo-id username/my-env
+```
+
+Your environment is now live at:
+- Web UI: https://username-my-env.hf.space/web
+- API Docs: https://username-my-env.hf.space/docs
+- Health: https://username-my-env.hf.space/health
+
+```bash
+curl https://openenv-echo-env.hf.space/health
+# {"status": "healthy"}
+```
+
+## Step 4: install the environment
+
+```bash
+uv pip install git+https://huggingface.co/spaces/openenv/echo_env
+```
+
+## Step 5: Run locally via Docker (optional)
+
+Pull and run the container from the HF registry, or open the [browser](https://huggingface.co/spaces/openenv/echo_env?docker=true):
+
+```bash
+# Pull from HF Spaces registry
+docker pull registry.hf.space/openenv-echo-env:latest
+
+# Run locally
+docker run -it -p 7860:7860 --platform=linux/amd64 \
+	registry.hf.space/openenv-echo-env:latest
+```
+
+Now connect to your local instance:
+
+```python
+import asyncio
+from echo_env import EchoEnv, EchoAction
+
+# Async (recommended)
+async def main():
+    async with EchoEnv(base_url="http://localhost:8000") as env:
+        result = await env.reset()
+        print(result.observation)
+        result = await env.step(EchoAction(message="Hello"))
+        print(result.observation)
+
+asyncio.run(main())
+
+# Sync (using .sync() wrapper)
+with EchoEnv(base_url="http://localhost:8000").sync() as env:
+    result = env.reset()
+    print(result.observation)
+    result = env.step(EchoAction(message="Hello"))
+    print(result.observation)
+```

tutorial/03-scaling.md

@@ -0,0 +1,457 @@
+# 3. How OpenEnv environments scale
+
+This section covers benchmarking and scaling OpenEnv environments.
+
+**Contents:**
+- [Provider Scaling](#provider-scaling)
+- [WebSocket-based Scaling](#websocket-based-scaling)
+- [Microservice Scaling](#microservice-scaling)
+- [Scaling Experiments](#scaling-experiments)
+
+---
+
+## Provider Scaling
+
+The easiest way to scale an OpenEnv environment is to use a `provider` these are abstractions based on runtimes like Uvicorn, Docker Swarm, or Kubernetes.
+
+```python
+from openenv.providers import UVProvider, DockerSwarmProvider, LocalDockerProvider
+
+docker_provider = LocalDockerProvider() # default
+uvicorn_provider = UVProvider() # python only
+swarm_provider = DockerSwarmProvider() 
+
+with EchoEnv.from_hub(
+    repo_id="openenv/echo-env", 
+    provider=swarm_provider, 
+    replicas=4,
+) as env:
+  result = env.reset()
+  result = env.step(EchoAction(message="Hello"))
+```
+
+## WebSocket-based Scaling
+
+OpenEnv uses WebSocket connections (`/ws`) instead of stateless HTTP for environment interactions. This design enables efficient scaling within a single container.
+
+### What are WebSockets?
+
+WebSocket is a communication protocol that provides a persistent, bidirectional connection between client and server. Unlike HTTP—where each request opens a new connection, sends data, receives a response, and closes—a WebSocket connection stays open for the duration of a session.
+
+![WebSocket vs HTTP](../images/websocket.png)
+
+For RL environments, this matters because a typical episode involves dozens to thousands of sequential `step()` calls. With HTTP, each step incurs TCP handshake overhead (~10-50ms). With WebSocket, messages are sent as lightweight frames (~0.1ms overhead) over the existing connection.
+
+Also, with HTTP, long running sessions require logic to manage session state, which is not necessary with WebSocket.
+
+### Multiple sessions per container
+
+With HTTP, maintaining session state requires cookies or session IDs with every request. Each isolated environment instance typically needs its own container:
+
+```
+HTTP approach: N parallel episodes → N containers
+```
+
+> [!NOTE]
+> This is completely fine (and ideal) for larger deployments where containers can be scaled. But if your resources are constrained, this add loads of overhead.
+
+With WebSocket, **one container handles many isolated sessions**. Each WebSocket connection gets its own environment instance server-side:
+
+```python
+# Single container serving multiple concurrent sessions
+# docker run -d -p 8000:8000 my-env:latest
+
+# Each client gets an isolated environment instance
+with MyEnv(base_url="http://localhost:8000") as env1:  # Session 1
+    result = env1.reset()
+    
+with MyEnv(base_url="http://localhost:8000") as env2:  # Session 2
+    result = env2.reset()
+    
+with MyEnv(base_url="http://localhost:8000") as env3:  # Session 3
+    result = env3.reset()
+```
+
+> [!NOTE]
+> This has its own advantages and disadvantages. For example: Separation of concerns and fault tolerance in environments like coding or terminal.
+
+### Server-side session state
+
+The server maintains environment state per WebSocket connection which means that the environment builder does not need to worry about session state.
+
+- No session IDs because Connection itself is the session
+- Automatic cleanup because Environment instance destroyed when connection closes
+- Isolation guaranteed because Each connection has dedicated state
+
+```python
+# Server creates new environment instance per WebSocket connection
+@app.websocket("/ws")
+async def websocket_endpoint(websocket: WebSocket):
+    env = MyEnvironment()  # Fresh instance per connection
+    await websocket.accept()
+    
+    while True:
+        data = await websocket.receive_json()
+        if data["type"] == "reset":
+            result = env.reset()
+        elif data["type"] == "step":
+            result = env.step(data["action"])
+        await websocket.send_json(result)
+```
+
+### Resource efficiency
+
+| Approach | Containers | Memory | Startup | Max parallel |
+|----------|------------|--------|---------|--------------|
+| HTTP (1 env = 1 container) | N | N × ~100MB | N × ~5s | Limited by containers |
+| WebSocket (N sessions = 1 container) | 1 | ~200MB | ~5s | Limited by `MAX_CONCURRENT_ENVS` |
+
+Configure session limits via environment variable:
+
+```bash
+docker run -d -p 8000:8000 -e MAX_CONCURRENT_ENVS=100 registry.hf.space/openenv-echo-env:latest
+```
+
+## Scaling a Single Container
+
+Before adding more containers, maximize the capacity of a single deployment. The key parameters are **workers** (CPU parallelism) and **MAX_CONCURRENT_ENVS** (session limit).
+
+### Uvicorn workers
+
+Each Uvicorn worker is a separate process that can handle requests independently. More workers = more CPU cores utilized.
+
+```bash
+# Clone and run locally
+git clone https://huggingface.co/spaces/burtenshaw/openenv-benchmark
+cd openenv-benchmark
+pip install -e .
+
+# Run with 8 workers
+WORKERS=8 uvicorn benchmark.server.app:app --host 0.0.0.0 --port 8000 --workers 8
+```
+
+The above example will use 8 workers and each worker will be able to handle 100 concurrent sessions. **For simple environments, like text games, it's possible to get to 2000 concurrent sessions with 8 workers.**
+
+> **Note:** More workers consume more memory. Each worker loads a full copy of the environment code.
+
+### Docker with environment variables
+
+Pass scaling parameters when starting the container:
+
+```bash
+# Pull from HF Spaces registry
+docker pull registry.hf.space/burtenshaw-openenv-benchmark:latest
+
+# Run with custom configuration
+docker run -d -p 8000:8000 \
+    -e WORKERS=8 \
+    -e MAX_CONCURRENT_ENVS=400 \
+    --name openenv-benchmark \
+    registry.hf.space/burtenshaw-openenv-benchmark:latest
+```
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `WORKERS` | 4 | Uvicorn worker processes |
+| `MAX_CONCURRENT_ENVS` | 100 | Max WebSocket sessions per worker |
+| `PORT` | 8000 | Server port |
+| `HOST` | 0.0.0.0 | Bind address |
+
+### HF Spaces configuration
+
+Now, let's deploy the environment to HF Spaces so that we can interact with the server from the client. Configure scaling via Space Settings > Variables:
+
+1. Go to your Space settings page
+2. Add environment variables:
+   - `WORKERS=4` (max 4 on free tier, 8 on CPU Upgrade)
+   - `MAX_CONCURRENT_ENVS=100`
+3. Restart the Space
+
+| Tier | vCPU | Recommended workers | Expected max batch (textarena) |
+|------|------|--------------------|--------------------|
+| CPU Basic (Free) | 2 | 2 | ~128 |
+| CPU Upgrade | 8 | 4-8 | ~512 |
+
+> **Limitation:** HF Spaces free users tier caps at ~128 concurrent sessions regardless of configuration. See [Scaling Experiments](#scaling-experiments) for measured limits. 
+
+### Scaling limits
+
+The experiments below found that even on larger instances, a single container eventually fails to scale and we need multiple containers to handle the load. For example, on a CPU Upgrade instance with 8 workers, the max batch was 1024 concurrent sessions:
+
+- Success rate drops to 92%
+- P99 latency exceeds 2× the expected step time
+- Connection errors increase under load
+
+When this happens, we need to scale to multiple containers and use a load balancer.
+
+For high-throughput workloads, scale horizontally by running multiple environment containers behind a load balancer.
+
+| Scenario | Recommended approach |
+|----------|---------------------|
+| Development / testing | Single container with WebSocket sessions |
+| Moderate load (< 100 concurrent) | Single container, increase `MAX_CONCURRENT_ENVS` |
+| High load (100+ concurrent) | Multiple containers + load balancer |
+| GPU environments | One container per GPU |
+
+We explored this in detail in the [Scaling Experiments](https://github.com/burtenshaw/openenv-scaling) repository.
+
+<details>
+<summary>Envoy configuration</summary>
+
+```yaml
+static_resources:
+  listeners:
+    - name: listener_0
+      address:
+        socket_address:
+          address: 0.0.0.0
+          port_value: 8080
+      filter_chains:
+        - filters:
+            - name: envoy.filters.network.http_connection_manager
+              typed_config:
+                "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+                stat_prefix: ingress_http
+                upgrade_configs:
+                  - upgrade_type: websocket
+                route_config:
+                  name: local_route
+                  virtual_hosts:
+                    - name: openenv_service
+                      domains: ["*"]
+                      routes:
+                        - match:
+                            prefix: "/"
+                          route:
+                            cluster: openenv_cluster
+                http_filters:
+                  - name: envoy.filters.http.router
+                    typed_config:
+                      "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
+
+  clusters:
+    - name: openenv_cluster
+      connect_timeout: 30s
+      type: STRICT_DNS
+      lb_policy: ROUND_ROBIN
+      load_assignment:
+        cluster_name: openenv_cluster
+        endpoints:
+          - lb_endpoints:
+              - endpoint:
+                  address:
+                    socket_address:
+                      address: host.docker.internal
+                      port_value: 8001
+              - endpoint:
+                  address:
+                    socket_address:
+                      address: host.docker.internal
+                      port_value: 8002
+              - endpoint:
+                  address:
+                    socket_address:
+                      address: host.docker.internal
+                      port_value: 8003
+              - endpoint:
+                  address:
+                    socket_address:
+                      address: host.docker.internal
+                      port_value: 8004
+```
+
+
+Start Envoy:
+
+```bash
+docker run -d \
+    -p 8080:8080 \
+    -v $(pwd)/envoy.yaml:/etc/envoy/envoy.yaml \
+    --add-host=host.docker.internal:host-gateway \
+    envoyproxy/envoy:v1.28.0
+```
+
+Connect through the load balancer:
+
+```python
+# Clients connect to Envoy, which distributes to backend containers
+with MyEnv(base_url="http://localhost:8080") as env:
+    result = env.reset()
+```
+
+</details>
+
+### Scaling expectations
+
+![Scaling Expectations](../images/scaling.png)
+
+| Setup | Containers | Sessions/container | Total capacity | Throughput |
+|-------|------------|-------------------|----------------|------------|
+| Single | 1 | 100 | 100 | ~100 req/s |
+| 4× containers | 4 | 100 | 400 | ~350 req/s |
+| 8× containers | 8 | 100 | 800 | ~600 req/s |
+
+> **Note:** Actual throughput depends on environment complexity and hardware. Benchmark your specific workload.
+
+## Experiments Results
+
+This section documents experiments measuring OpenEnv scaling characteristics across five infrastructure configurations. Full experiment data and code available at [burtenshaw/openenv-scaling](https://github.com/burtenshaw/openenv-scaling).
+
+### Experiment setup
+
+**Benchmark environment:** A minimal OpenEnv environment with configurable wait time (simulates computation). Each `step()` call sleeps for the specified duration, isolating infrastructure overhead from environment logic.
+
+**Infrastructure tested:**
+
+| Infrastructure | Cores | Configuration |
+|----------------|-------|---------------|
+| local-uvicorn | 8 | Direct Uvicorn, 8 workers |
+| local-docker | 8 | Docker container from HF Spaces image |
+| hf-spaces | 2 | HF Spaces free tier (cpu-basic) |
+| slurm-single | 48 | Single AWS HPC node |
+| slurm-multi | 96 | Two AWS HPC nodes + Envoy load balancer |
+
+**Protocol:** WebSocket (`/ws`) and HTTP (`/reset`, `/step`) compared where available.
+
+**Metrics:**
+- **Max batch:** Largest concurrent request count with ≥95% success rate
+- **Batch/core:** Max batch divided by available cores (efficiency metric)
+- **P99 latency:** 99th percentile total request time
+- **RPS:** Requests per second at max batch
+
+### Results summary
+
+| Infrastructure | Max Batch (WS) | Cores | Batch/Core | P99 Latency | RPS |
+|----------------|----------------|-------|------------|-------------|-----|
+| slurm-multi | 16,384 | 96 | 170.7 | 29.8s | 518 |
+| local-uvicorn | 2,048 | 8 | 256.0 | 1.97s | 932 |
+| local-docker | 2,048 | 8 | 256.0 | 2.90s | 682 |
+| slurm-single | 512 | 48 | 10.7 | 1.45s | 358 |
+| hf-spaces | 128 | 2 | 64.0 | 2.68s | 48 |
+
+All results measured with `wait=10.0s` step duration.
+
+![Max Batch Comparison](https://raw.githubusercontent.com/burtenshaw/openenv-scaling/main/experiments/reports/figures/max_batch_comparison.png)
+*Maximum batch size by infrastructure (95% success threshold)*
+
+### Finding 1: Local deployments have highest per-core efficiency
+
+Single instance of Python and Docker both achieve **256 concurrent sessions per core**—the highest efficiency observed. With 8 workers, both reach 2,048 concurrent sessions before degradation begins.
+
+This makes sense because the environment is running in a single process and the overhead of the environment is relatively low. But it's ideal for hackers and developers who want to test their environment quickly or train on a single machine.
+
+| Batch Size | Success Rate | P99 Latency | Notes |
+|------------|--------------|-------------|-------|
+| 32 | 100% | 1.05s | Perfect scaling |
+| 128 | 100% | 1.07s | Perfect scaling |
+| 512 | 100% | 1.33s | Perfect scaling |
+| 2,048 | 96.5% | 1.97s | Max reliable batch |
+| 4,096 | 63.8% | 3.20s | Connection failures begin |
+| 8,192 | 36.9% | 5.75s | Above capacity |
+
+Beyond 2,048 concurrent connections, success rate drops sharply. The failure mode is connection rejection, not timeout—the server saturates its connection pool.
+
+![Batch Per Core](https://raw.githubusercontent.com/burtenshaw/openenv-scaling/main/experiments/reports/figures/batch_per_core.png)
+*Per-core efficiency comparison across infrastructures*
+
+### Finding 2: HF Spaces works reliably up to 128 concurrent sessions
+
+HF Spaces free tier (cpu-basic) provides 2 workers and achieves 128 concurrent WebSocket sessions with 100% success. This translates to **64 sessions per core**.
+
+**HF Spaces scaling behavior (WebSocket):**
+
+| Batch Size | Success Rate | P99 Latency | Notes |
+|------------|--------------|-------------|-------|
+| 1 | 100% | 1.64s | Baseline |
+| 32 | 100% | 1.80s | Perfect scaling |
+| 64 | 100% | 2.14s | Perfect scaling |
+| 128 | 100% | 2.68s | Max reliable batch |
+| 256 | ~33% | 4.41s | Inconsistent (some runs 0%, some 100%) |
+| 512 | 0% | — | Complete failure |
+
+At 256 concurrent connections, results become unstable. At 512+, connections fail entirely due to HF Spaces connection limits.
+
+**HTTP mode does not work on HF Spaces.** The `/reset` and `/step` HTTP endpoints are not accessible on the deployed Space—all HTTP requests fail. Use WebSocket mode exclusively.
+
+### Finding 3: Multi-node scaling works
+
+Multi-node SLURM (96 cores across 2 nodes) achieves **16,384 concurrent sessions** with 100% success rate—the highest absolute throughput tested.
+
+**SLURM multi-node scaling behavior:**
+
+| Batch Size | Success Rate | P99 Latency | Notes |
+|------------|--------------|-------------|-------|
+| 32 | 100% | 1.05s | Perfect scaling |
+| 512 | 100% | 1.59s | Perfect scaling |
+| 2,048 | 100% | 3.48s | Perfect scaling |
+| 4,096 | 100% | 6.97s | Perfect scaling |
+| 8,192 | 100% | 13.7s | Perfect scaling |
+| 16,384 | 100% | 29.8s | Max tested batch |
+
+The batch/core ratio (170.7) is lower than local deployments (256) but provides the highest absolute capacity for large-scale workloads.
+
+![Scaling Comparison](https://raw.githubusercontent.com/burtenshaw/openenv-scaling/main/experiments/reports/figures/scaling_comparison.png)
+
+*Multi-node vs single-node scaling behavior*
+
+### Latency breakdown
+
+At max load (`wait=1.0s`), latency breaks down as:
+
+| Infrastructure | Connect P50 | Reset P50 | Step P50 | Total P99 |
+|----------------|-------------|-----------|----------|-----------|
+| slurm-single | 0.26s | 0.04s | 1.00s | 1.33s |
+| local-uvicorn | 0.58s | 0.08s | 1.05s | 1.95s |
+| hf-spaces | 0.79s | 0.10s | 1.10s | 2.48s |
+| local-docker | 1.38s | 0.19s | 1.05s | 2.90s |
+| slurm-multi | 17.5s | 2.25s | 2.42s | 26.3s |
+
+**Observations:**
+- **Step latency** is consistent across infrastructures (~1.0s for 1.0s wait), confirming the benchmark measures infrastructure overhead accurately
+- **Connect latency** varies significantly—local Docker shows higher connect time at load (1.38s), likely due to container networking
+- **Multi-node has high connect latency** (17.5s) at 16,384 batch due to queuing at the load balancer; this is the cost of handling 16× more connections than single-node
+
+![Latency Heatmap](https://raw.githubusercontent.com/burtenshaw/openenv-scaling/main/experiments/reports/figures/latency_heatmap.png)
+*P99 latency across configurations and batch sizes*
+
+![Scaling Curves](https://raw.githubusercontent.com/burtenshaw/openenv-scaling/main/experiments/reports/figures/scaling_curves.png)
+*Success rate vs batch size for all infrastructures*
+
+### Test methodology
+
+```bash
+# Clone benchmark environment
+git clone https://huggingface.co/spaces/burtenshaw/openenv-scaling
+cd openenv-scaling
+
+# Run scaling test
+python tests/test_scaling.py \
+    --url http://localhost:8000 \
+    --requests-grid 32,128,512,2048,4096,8192,16384 \
+    --wait-grid 1.0,5.0,10.0 \
+    --reps 3 \
+    --mode ws \
+    --output-dir experiments/results/
+```
+
+Each configuration was tested with 3 repetitions. Max batch is defined as the largest batch size achieving ≥95% success rate across all repetitions.
+
+---
+
+## Summary
+
+| Infrastructure | Best for | Max concurrent | Batch/core |
+|----------------|----------|----------------|------------|
+| local-uvicorn | Development, <2K sessions | 2,048 | 256 |
+| local-docker | Same as uvicorn, containerized | 2,048 | 256 |
+| hf-spaces | Demos, moderate load | 128 | 64 |
+| slurm-single | HPC, single-node jobs | 512 | 10.7 |
+| slurm-multi | Large-scale training | 16,384 | 170.7 |
+
+**Recommendations:**
+
+1. **For development and moderate workloads (<2,000 concurrent):** Use single node Uvicorn or Docker depending software environment. These provide the best per-core efficiency (256 sessions/core).
+
+2. **For demos, testing, and published environments:** HF Spaces free tier works reliably up to 128 concurrent sessions.
+
+3. **For large-scale training (>2,000 concurrent):** Deploy multi-node with proper load balancing. Expect ~170 sessions per core, but much higher absolute throughput.

tutorial/04-training.md

@@ -0,0 +1,632 @@
+# OpenEnv Wordle with GRPO using TRL
+
+[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://github.com/huggingface/trl/blob/main/examples/notebooks/openenv_wordle_grpo.ipynb)
+
+![trl banner](https://huggingface.co/datasets/trl-lib/documentation-images/resolve/main/trl_banner_dark.png)
+
+With [**Transformers Reinforcement Learning (TRL)**](https://github.com/huggingface/trl), you can train a model that learns to **play Wordle**, a word-guessing game, through interaction and reinforcement.
+
+- [TRL GitHub Repository](https://github.com/huggingface/trl)
+- [Official TRL Examples](https://huggingface.co/docs/trl/example_overview)
+- [Community Tutorials](https://huggingface.co/docs/trl/community_tutorials)
+- [OpenEnv](https://github.com/meta-pytorch/OpenEnv)
+
+An **agentic environment** is a setting where a model can take actions, observe outcomes, and adjust its behavior based on feedback, similar to how humans learn from trial and error.
+In this case, the agent interacts with the **Wordle** environment through the [**OpenEnv**](https://github.com/meta-pytorch/OpenEnv) framework, which standardizes multi-agent and RL-style text environments.
+
+[Wordle](https://en.wikipedia.org/wiki/Wordle) is a popular word puzzle where the player must guess a secret five-letter word within six tries.
+After each guess, feedback indicates whether each letter is:
+
+- 🟩 **Correct and in the right position**
+- 🟨 **Present but in the wrong position**
+- ⬛ **Not in the word**
+
+This feedback loop makes Wordle a perfect environment for **RL with LLMs**, where the goal is to maximize the probability of guessing the correct word efficiently.
+
+We will fine-tune a model using **GRPO** (Group Relative Policy Optimization) via TRL.
+The agent will:
+
+1. Generate guesses based on the game state and feedback.
+2. Receive structured feedback from the environment after each guess.
+3. Learn to improve its guessing strategy over time through reward signals.
+
+---
+
+## Install dependencies
+
+We will start by installing **TRL**, which automatically includes the main dependencies like **Transformers**.
+We will also install the **OpenEnv** framework (for the environment), **trackio** (for logging and monitoring training runs), and **vLLM** (for efficient generation).
+
+```python
+!pip install -Uq git+https://github.com/huggingface/trl.git git+https://github.com/meta-pytorch/OpenEnv.git trackio vllm==0.10.2 bitsandbytes
+```
+
+---
+
+## Log in to Hugging Face
+
+Log in to your **Hugging Face** account to save your fine-tuned model, track your experiment results directly on the Hub or access gated models. You can find your **access token** on your [account settings page](https://huggingface.co/settings/tokens).
+
+```python
+from huggingface_hub import notebook_login
+
+notebook_login()
+```
+
+---
+
+## Initialize the Environment
+
+Let us begin by setting up the environment that will be used during training.
+For this task, we will rely on the **TextArena** environment from **OpenEnv**, which exposes a familiar Gymnasium-style API (`reset()`, `step()`, etc.) to simplify interaction.
+
+In this example, we will connect to the hosted environment at [burtenshaw/textarena](https://huggingface.co/spaces/burtenshaw/textarena).
+For production use or custom configurations, we **strongly recommend** running the environment locally via Docker. The hosted versions on the Hub currently have limited concurrency support, so duplicating the Space to your own account is the preferred approach in those cases.
+
+For more information, refer to the [TRL-OpenEnv documentation](https://huggingface.co/docs/trl/main/en/openenv).
+
+```python
+from envs.textarena_env import TextArenaEnv
+
+textarena_url = "https://burtenshaw-textarena.hf.space" # Duplicate the Space and update this!
+env = TextArenaEnv(base_url=textarena_url)
+```
+
+---
+
+## Init model and tokenizer
+
+We will use [Qwen/Qwen3-1.7B](https://huggingface.co/Qwen/Qwen3-1.7B), a lightweight instruction-tuned model that works well for quick experiments.
+Despite its small size, it can still learn interesting strategies during fine-tuning.
+If you have stronger hardware, you can easily scale up to larger models.
+
+```python
+from transformers import AutoTokenizer
+
+model_name = "Qwen/Qwen3-1.7B"
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+tokenizer.pad_token = tokenizer.eos_token
+```
+
+---
+
+## Rollout function with helpers
+
+The **rollout function** defines how the agent interacts with the environment during GRPO training.
+It is responsible for generating model completions, collecting feedback (rewards), and returning all necessary information for optimization.
+
+In this setup:
+
+- The function is called automatically by the **GRPOTrainer** during each training step.
+- It uses the trainer's built-in `generate_rollout_completions()` method for efficient generation with vLLM in colocate mode.
+- Each rollout represents a full interaction loop. The model guesses, receives feedback from Wordle, and updates based on reward signals.
+
+### System Prompt
+
+First, we define the `system_prompt` that guides the model's behavior as an expert Wordle solver with strategic reasoning and structured responses.
+
+```python
+system_prompt = """
+You are an expert Wordle solver with deep knowledge of English vocabulary, letter frequency patterns, and optimal guessing strategies.
+
+## GAME RULES
+
+1. The target is a 5-letter English word
+2. You have 6 attempts to guess the correct word
+3. After each guess, you receive color-coded feedback:
+   - GREEN: Letter is correct and in the correct position
+   - YELLOW: Letter is in the word but in the wrong position
+   - GRAY: Letter is not in the word at all
+4. All guesses must be valid 5-letter English words
+5. You cannot reuse a word you've already guessed
+
+## RESPONSE FORMAT
+
+Only respond with your next guess in square brackets, e.g., [crane].
+
+## STRATEGIC APPROACH
+
+Do not repeat the same guess twice.
+
+### Opening Strategy
+- Start with words rich in common vowels (A, E, I, O, U) and consonants (R, S, T, L, N)
+- Optimal starters: CRANE, SLATE, STARE, AROSE, IRATE
+
+### Mid-Game Strategy
+- Use confirmed GREEN letters in their correct positions
+- Place YELLOW letters in different positions than where they appeared
+- Eliminate GRAY letters from consideration
+
+## YOUR GOAL
+
+Solve the Wordle in as few guesses as possible by strategically using feedback to eliminate impossible words and narrow down the solution space efficiently.
+"""
+```
+
+### Rollout Function
+
+```python
+def rollout_func(prompts, trainer=None):
+    """
+    Rollout function for GRPO training with environment interaction.
+    """
+    episode_prompt_ids = []
+    episode_completion_ids = []
+    episode_logprobs = []
+    correctness_rewards = []
+    green_rewards = []
+    yellow_rewards = []
+    repetition_rewards = []
+
+    for prompt_text in prompts:
+        episode = rollout_once(
+            trainer=trainer,
+            env=env,
+            tokenizer=tokenizer,
+            dataset_prompt=prompt_text,
+            system_prompt=system_prompt,
+            max_turns=6,
+        )
+        episode_prompt_ids.append(episode["prompt_ids"])
+        episode_completion_ids.append(episode["completion_ids"])
+        episode_logprobs.append(episode["logprobs"])
+        correctness_rewards.append(episode["correct_reward"])
+        green_rewards.append(episode["green_reward"])
+        yellow_rewards.append(episode["yellow_reward"])
+        repetition_rewards.append(episode["repetition_reward"])
+
+    return {
+        "prompt_ids": episode_prompt_ids,
+        "completion_ids": episode_completion_ids,
+        "logprobs": episode_logprobs,
+        "correct_reward": correctness_rewards,
+        "green_reward": green_rewards,
+        "yellow_reward": yellow_rewards,
+        "repetition_reward": repetition_rewards,
+    }
+```
+
+---
+
+## Define rollout_once
+
+The `rollout_once` function runs **one full interaction loop** between the model and the Wordle environment using the trainer's generation method.
+
+```python
+from collections import defaultdict
+from envs.textarena_env import TextArenaAction
+from envs.textarena_env.rewards import extract_feedback_counts, extract_guess, extract_wordle_feedback
+from trl.experimental.openenv import generate_rollout_completions
+
+
+def rollout_once(trainer, env, tokenizer, dataset_prompt, system_prompt, max_turns):
+    """
+    Execute one full Wordle episode with the model.
+    """
+    result = env.reset()
+    observation = result.observation
+
+    prompt_ids = []
+    completion_ids = []
+    logprobs = []
+    raw_rewards = []
+    green_scores = []
+    yellow_scores = []
+    repetition_scores = []
+    correct_scores = []
+    guess_counts = defaultdict(int)
+
+    for _turn in range(max_turns):
+        if result.done:
+            break
+
+        base_prompt = observation.prompt or dataset_prompt
+        user_prompt = make_user_prompt(base_prompt, observation.messages)
+        messages = [
+            {"role": "system", "content": system_prompt},
+            {"role": "user", "content": user_prompt},
+        ]
+        prompt_text = tokenizer.apply_chat_template(
+            messages,
+            add_generation_prompt=True,
+            tokenize=False,
+            enable_thinking=False,
+        )
+
+        rollout_outputs = generate_rollout_completions(trainer, [prompt_text])[0]
+        prompt_ids.extend(rollout_outputs["prompt_ids"])
+        completion_ids.extend(rollout_outputs["completion_ids"])
+        logprobs.extend(rollout_outputs["logprobs"])
+        completion_text = rollout_outputs.get("text") or tokenizer.decode(
+            rollout_outputs["completion_ids"], skip_special_tokens=True
+        )
+
+        guess = extract_guess(completion_text)
+        result = env.step(TextArenaAction(message=guess))
+        raw_rewards.append(float(result.reward or 0.0))
+        observation = result.observation
+        correct_score = float(result.reward or 0.0)
+        feedback = extract_wordle_feedback(observation)
+
+        previous_occurrences = guess_counts[guess]
+        repetition_score = scale_repetition_score(previous_occurrences, len(guess_counts))
+        guess_counts[guess] += 1
+
+        if not feedback:
+            green_score = 0.0
+            yellow_score = 0.0
+        else:
+            green_count, yellow_count = extract_feedback_counts(feedback)
+            green_score = green_count / 5.0
+            yellow_score = yellow_count / 5.0
+
+        repetition_scores.append(repetition_score)
+        green_scores.append(green_score)
+        yellow_scores.append(yellow_score)
+        correct_scores.append(correct_score)
+
+    correct_reward_value = correct_scores[-1] if correct_scores else (raw_rewards[-1] if raw_rewards else 0.0)
+
+    return {
+        "prompt_ids": prompt_ids,
+        "completion_ids": completion_ids,
+        "logprobs": logprobs,
+        "raw_rewards": raw_rewards,
+        "correct_reward": correct_reward_value,
+        "green_reward": green_scores[-1] if green_scores else 0.0,
+        "yellow_reward": yellow_scores[-1] if yellow_scores else 0.0,
+        "repetition_reward": repetition_scores[-1] if repetition_scores else 0.0,
+    }
+```
+
+---
+
+## Helper functions
+
+```python
+def make_user_prompt(prompt_text, messages):
+    """Builds a structured user prompt combining the task description and message history"""
+    history = format_history(messages)
+    prompt_section = prompt_text.strip() if prompt_text.strip() else "Wordle-v0"
+    history_section = history if history else "[PROMPT] Awaiting first feedback."
+    return (
+        f"Game prompt:\n{prompt_section}\n\n"
+        f"Conversation so far:\n{history_section}\n\n"
+        "Reply with your next guess enclosed in square brackets."
+    )
+
+def format_history(messages):
+    """Formats the message history with tags for clear conversational context"""
+    lines = []
+    for message in messages:
+        tag = message.category or "MESSAGE"
+        content = message.content.strip()
+        if not content:
+            continue
+        lines.append(f"[{tag}] {content}")
+    return "\n".join(lines)
+
+def scale_repetition_score(previous_occurrences, max_occurrences):
+    """Scale the repetition score based on the number of previous occurrences from 0 to 1"""
+    if max_occurrences == 0:
+        return 0.0
+    return (max_occurrences - previous_occurrences) / max_occurrences
+```
+
+---
+
+## Define reward functions
+
+```python
+def reward_correct(completions, **kwargs):
+    rewards = kwargs.get("correct_reward") if kwargs else None
+    if rewards is None:
+        return [0.0 for _ in completions]
+    return [float(r) for r in rewards]
+
+
+def reward_greens(completions, **kwargs):
+    rewards = kwargs.get("green_reward") if kwargs else None
+    if rewards is None:
+        return [0.0 for _ in completions]
+    return [float(r) for r in rewards]
+
+
+def reward_yellows(completions, **kwargs):
+    rewards = kwargs.get("yellow_reward") if kwargs else None
+    if rewards is None:
+        return [0.0 for _ in completions]
+    return [float(r) for r in rewards]
+
+
+def reward_repetition(completions, **kwargs):
+    rewards = kwargs.get("repetition_reward") if kwargs else None
+    if rewards is None:
+        return [0.0 for _ in completions]
+    return [float(r) for r in rewards]
+```
+
+---
+
+## Create dataset
+
+```python
+from datasets import Dataset
+
+dataset_size = 1000
+dataset_prompt = "Play Wordle like an expert."
+
+dataset = Dataset.from_dict({"prompt": [dataset_prompt] * dataset_size})
+```
+
+---
+
+## Set GRPO Config
+
+```python
+from trl import GRPOConfig
+
+output_dir = "wordle-grpo-Qwen3-1.7B"
+
+grpo_config = GRPOConfig(
+    num_train_epochs = 1,
+    learning_rate = 5e-6,
+    gradient_accumulation_steps = 64,
+    per_device_train_batch_size = 1,
+    warmup_steps = 20,
+    num_generations = 2,
+    max_completion_length = 8,
+    max_prompt_length = 1400,
+    use_vllm = True,
+    vllm_mode = "colocate",
+    vllm_gpu_memory_utilization = 0.1,
+    output_dir = output_dir,
+    report_to="trackio",
+    trackio_space_id = output_dir,
+    logging_steps = 1,
+    save_steps = 10,
+    gradient_checkpointing = True,
+    gradient_checkpointing_kwargs = {"use_reentrant": False},
+    push_to_hub = True,
+)
+```
+
+---
+
+## Create GRPOTrainer and start training
+
+```python
+from trl import GRPOTrainer
+
+trainer = GRPOTrainer(
+    model=model_name,
+    processing_class=tokenizer,
+    reward_funcs=[
+        reward_correct,
+        reward_greens,
+        reward_yellows,
+        reward_repetition,
+    ],
+    train_dataset=dataset,
+    args=grpo_config,
+    rollout_func=rollout_func,
+)
+```
+
+### Memory stats before training
+
+```python
+import torch
+gpu_stats = torch.cuda.get_device_properties(0)
+start_gpu_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3)
+max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3)
+
+print(f"GPU = {gpu_stats.name}. Max memory = {max_memory} GB.")
+print(f"{start_gpu_memory} GB of memory reserved.")
+```
+
+**Output:**
+```
+GPU = NVIDIA A100-SXM4-40GB. Max memory = 39.557 GB.
+10.516 GB of memory reserved.
+```
+
+### Train!
+
+```python
+trainer_stats = trainer.train()
+```
+
+**Training Progress:**
+
+| Step | Training Loss |
+|------|---------------|
+| 1 | 0.008300 |
+| 2 | 0.001900 |
+| 3 | 0.015100 |
+| 4 | 0.008700 |
+| 5 | 0.009800 |
+| 6 | 0.006700 |
+| 7 | 0.006100 |
+| 8 | 0.004400 |
+| 9 | -0.002100 |
+| 10 | 0.007500 |
+| 11 | 0.008400 |
+| 12 | 0.008000 |
+| 13 | 0.007800 |
+| 14 | -0.002400 |
+| 15 | -0.003200 |
+| 16 | -0.006000 |
+| 17 | -0.008300 |
+| 18 | -0.011000 |
+| 19 | -0.004200 |
+| 20 | -0.001700 |
+| 21 | -0.004100 |
+| 22 | -0.011600 |
+| 23 | -0.006400 |
+| 24 | -0.009100 |
+| 25 | 0.003200 |
+| 26 | 0.005100 |
+| 27 | -0.002800 |
+| 28 | 0.001400 |
+| 29 | 0.011500 |
+| 30 | -0.010500 |
+| 31 | -0.006400 |
+
+### Memory stats after training
+
+```python
+used_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3)
+used_memory_for_training = round(used_memory - start_gpu_memory, 3)
+used_percentage = round(used_memory / max_memory * 100, 3)
+training_memory_percentage = round(used_memory_for_training / max_memory * 100, 3)
+
+print(f"{trainer_stats.metrics['train_runtime']} seconds used for training.")
+print(f"{round(trainer_stats.metrics['train_runtime']/60, 2)} minutes used for training.")
+print(f"Peak reserved memory = {used_memory} GB.")
+print(f"Peak reserved memory for training = {used_memory_for_training} GB.")
+print(f"Peak reserved memory % of max memory = {used_percentage} %.")
+print(f"Peak reserved memory for training % of max memory = {training_memory_percentage} %.")
+```
+
+**Output:**
+```
+5231.7046 seconds used for training.
+87.2 minutes used for training.
+Peak reserved memory = 36.68 GB.
+Peak reserved memory for training = 26.164 GB.
+Peak reserved memory % of max memory = 92.727 %.
+Peak reserved memory for training % of max memory = 66.143 %.
+```
+
+### Save and push to Hub
+
+```python
+env.close()
+trainer.save_model(output_dir)
+trainer.push_to_hub()
+```
+
+---
+
+## Load the Fine-Tuned Model and Run Inference
+
+```python
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+model_name = "sergiopaniego/wordle-grpo-Qwen3-1.7B" # Replace with your HF username
+
+fine_tuned_model = AutoModelForCausalLM.from_pretrained(model_name, dtype="auto", device_map="auto")
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+```
+
+```python
+MAX_TURNS=6
+
+def play_wordle(env, model, tokenizer):
+    result = env.reset()
+    observation = result.observation
+
+    print("Initial Prompt:\n" + observation.prompt)
+
+    for turn in range(MAX_TURNS):
+        if result.done:
+            break
+
+        user_prompt = make_user_prompt(observation.prompt, observation.messages)
+        messages = [
+            {"role": "system", "content": system_prompt},
+            {"role": "user", "content": user_prompt},
+        ]
+        prompt_text = tokenizer.apply_chat_template(
+            messages,
+            add_generation_prompt=True,
+            tokenize=False,
+            enable_thinking=False,
+        )
+
+        model_inputs = tokenizer([prompt_text], return_tensors="pt").to(model.device)
+
+        generated_ids = model.generate(
+            **model_inputs,
+            max_new_tokens=512
+        )
+        output_ids = generated_ids[0][len(model_inputs.input_ids[0]):]
+
+        generated_text = tokenizer.decode(output_ids, skip_special_tokens=True)
+        guess = extract_guess(generated_text)
+
+        print(f"\nTurn {turn}: model replied with -> {generated_text}")
+        print(f"   Parsed guess: {guess}")
+
+        result = env.step(TextArenaAction(message=guess))
+        observation = result.observation
+
+        print("   Feedback messages:")
+        for message in observation.messages:
+            print(f"     [{message.category}] {message.content}")
+
+    print("\nGame finished")
+    print(f"   Reward: {result.reward}")
+    print(f"   Done: {result.done}")
+```
+
+### Let us play the game!
+
+```python
+try:
+    play_wordle(env, fine_tuned_model, tokenizer)
+finally:
+    env.close()
+```
+
+**Output:**
+```
+Initial Prompt:
+You are Player 0 in Wordle.
+A secret 5-letter word has been chosen. You have 6 attempts to guess it.
+For each guess, wrap your word in square brackets (e.g., [apple]).
+Feedback for each letter will be given as follows:
+  - G (green): correct letter in the correct position
+  - Y (yellow): letter exists in the word but in the wrong position
+  - X (wrong): letter is not in the word
+Enter your guess to begin.
+
+Turn 0: model replied with -> [crane]
+   Parsed guess: [crane]
+   Feedback messages:
+     [MESSAGE] [crane]
+     [MESSAGE] Player 0 submitted [crane].
+Feedback:
+C R A N E
+X Y X X X
+
+You have 5 guesses left.
+
+Turn 1: model replied with -> [spare]
+   Parsed guess: [spare]
+   Feedback messages:
+     [MESSAGE] [spare]
+     [MESSAGE] Player 0 submitted [spare].
+Feedback:
+C R A N E
+X Y X X X
+
+S P A R E
+G X X G X
+
+You have 4 guesses left.
+
+...
+
+Game finished
+   Reward: 0.0
+   Done: True
+```
+
+> **Note:** The model has learned some good opening strategies (starting with "crane", then "spare"), but still tends to repeat guesses. This is a common challenge in RL training that can be improved with:
+>
+> - Longer training runs
+> - Stronger repetition penalties
+> - Better reward shaping
+> - Larger models

validate.sh

@@ -0,0 +1,185 @@
+#!/usr/bin/env bash
+#
+# validate-submission.sh — OpenEnv Submission Validator
+#
+# Checks that your HF Space is live, Docker image builds, and openenv validate passes.
+#
+# Prerequisites:
+#   - Docker:       https://docs.docker.com/get-docker/
+#   - openenv-core: pip install openenv-core
+#   - curl (usually pre-installed)
+#
+# Run:
+#   curl -fsSL https://raw.githubusercontent.com/<owner>/<repo>/main/scripts/validate-submission.sh | bash -s -- <ping_url> [repo_dir]
+#
+#   Or download and run locally:
+#     chmod +x validate-submission.sh
+#     ./validate-submission.sh <ping_url> [repo_dir]
+#
+# Arguments:
+#   ping_url   Your HuggingFace Space URL (e.g. https://your-space.hf.space)
+#   repo_dir   Path to your repo (default: current directory)
+#
+# Examples:
+#   ./validate-submission.sh https://my-team.hf.space
+#   ./validate-submission.sh https://my-team.hf.space ./my-repo
+#
+
+set -uo pipefail
+
+DOCKER_BUILD_TIMEOUT=600
+if [ -t 1 ]; then
+  RED='\033[0;31m'
+  GREEN='\033[0;32m'
+  YELLOW='\033[1;33m'
+  BOLD='\033[1m'
+  NC='\033[0m'
+else
+  RED='' GREEN='' YELLOW='' BOLD='' NC=''
+fi
+
+run_with_timeout() {
+  local secs="$1"; shift
+  if command -v timeout &>/dev/null; then
+    timeout "$secs" "$@"
+  elif command -v gtimeout &>/dev/null; then
+    gtimeout "$secs" "$@"
+  else
+    "$@" &
+    local pid=$!
+    ( sleep "$secs" && kill "$pid" 2>/dev/null ) &
+    local watcher=$!
+    wait "$pid" 2>/dev/null
+    local rc=$?
+    kill "$watcher" 2>/dev/null
+    wait "$watcher" 2>/dev/null
+    return $rc
+  fi
+}
+
+portable_mktemp() {
+  local prefix="${1:-validate}"
+  mktemp "${TMPDIR:-/tmp}/${prefix}-XXXXXX" 2>/dev/null || mktemp
+}
+
+CLEANUP_FILES=()
+cleanup() { rm -f "${CLEANUP_FILES[@]+"${CLEANUP_FILES[@]}"}"; }
+trap cleanup EXIT
+
+PING_URL="${1:-}"
+REPO_DIR="${2:-.}"
+
+if [ -z "$PING_URL" ]; then
+  printf "Usage: %s <ping_url> [repo_dir]\n" "$0"
+  printf "\n"
+  printf "  ping_url   Your HuggingFace Space URL (e.g. https://your-space.hf.space)\n"
+  printf "  repo_dir   Path to your repo (default: current directory)\n"
+  exit 1
+fi
+
+if ! REPO_DIR="$(cd "$REPO_DIR" 2>/dev/null && pwd)"; then
+  printf "Error: directory '%s' not found\n" "${2:-.}"
+  exit 1
+fi
+PING_URL="${PING_URL%/}"
+export PING_URL
+PASS=0
+
+log()  { printf "[%s] %b\n" "$(date -u +%H:%M:%S)" "$*"; }
+pass() { log "${GREEN}PASSED${NC} -- $1"; PASS=$((PASS + 1)); }
+fail() { log "${RED}FAILED${NC} -- $1"; }
+hint() { printf "  ${YELLOW}Hint:${NC} %b\n" "$1"; }
+stop_at() {
+  printf "\n"
+  printf "${RED}${BOLD}Validation stopped at %s.${NC} Fix the above before continuing.\n" "$1"
+  exit 1
+}
+
+printf "\n"
+printf "${BOLD}========================================${NC}\n"
+printf "${BOLD}  OpenEnv Submission Validator${NC}\n"
+printf "${BOLD}========================================${NC}\n"
+log "Repo:     $REPO_DIR"
+log "Ping URL: $PING_URL"
+printf "\n"
+
+log "${BOLD}Step 1/3: Pinging HF Space${NC} ($PING_URL/reset) ..."
+
+CURL_OUTPUT=$(portable_mktemp "validate-curl")
+CLEANUP_FILES+=("$CURL_OUTPUT")
+HTTP_CODE=$(curl -s -o "$CURL_OUTPUT" -w "%{http_code}" -X POST \
+  -H "Content-Type: application/json" -d '{}' \
+  "$PING_URL/reset" --max-time 30 2>"$CURL_OUTPUT" || printf "000")
+
+if [ "$HTTP_CODE" = "200" ]; then
+  pass "HF Space is live and responds to /reset"
+elif [ "$HTTP_CODE" = "000" ]; then
+  fail "HF Space not reachable (connection failed or timed out)"
+  hint "Check your network connection and that the Space is running."
+  hint "Try: curl -s -o /dev/null -w '%%{http_code}' -X POST $PING_URL/reset"
+  stop_at "Step 1"
+else
+  fail "HF Space /reset returned HTTP $HTTP_CODE (expected 200)"
+  hint "Make sure your Space is running and the URL is correct."
+  hint "Try opening $PING_URL in your browser first."
+  stop_at "Step 1"
+fi
+
+log "${BOLD}Step 2/3: Running docker build${NC} ..."
+
+if ! command -v docker &>/dev/null; then
+  fail "docker command not found"
+  hint "Install Docker: https://docs.docker.com/get-docker/"
+  stop_at "Step 2"
+fi
+
+if [ -f "$REPO_DIR/Dockerfile" ]; then
+  DOCKER_CONTEXT="$REPO_DIR"
+elif [ -f "$REPO_DIR/server/Dockerfile" ]; then
+  DOCKER_CONTEXT="$REPO_DIR/server"
+else
+  fail "No Dockerfile found in repo root or server/ directory"
+  stop_at "Step 2"
+fi
+
+log "  Found Dockerfile in $DOCKER_CONTEXT"
+
+BUILD_OK=false
+BUILD_OUTPUT=$(run_with_timeout "$DOCKER_BUILD_TIMEOUT" docker build "$DOCKER_CONTEXT" 2>&1) && BUILD_OK=true
+
+if [ "$BUILD_OK" = true ]; then
+  pass "Docker build succeeded"
+else
+  fail "Docker build failed (timeout=${DOCKER_BUILD_TIMEOUT}s)"
+  printf "%s\n" "$BUILD_OUTPUT" | tail -20
+  stop_at "Step 2"
+fi
+
+log "${BOLD}Step 3/3: Running openenv validate${NC} ..."
+
+if ! command -v openenv &>/dev/null; then
+  fail "openenv command not found"
+  hint "Install it: pip install openenv-core"
+  stop_at "Step 3"
+fi
+
+VALIDATE_OK=false
+VALIDATE_OUTPUT=$(cd "$REPO_DIR" && openenv validate 2>&1) && VALIDATE_OK=true
+
+if [ "$VALIDATE_OK" = true ]; then
+  pass "openenv validate passed"
+  [ -n "$VALIDATE_OUTPUT" ] && log "  $VALIDATE_OUTPUT"
+else
+  fail "openenv validate failed"
+  printf "%s\n" "$VALIDATE_OUTPUT"
+  stop_at "Step 3"
+fi
+
+printf "\n"
+printf "${BOLD}========================================${NC}\n"
+printf "${GREEN}${BOLD}  All 3/3 checks passed!${NC}\n"
+printf "${GREEN}${BOLD}  Your submission is ready to submit.${NC}\n"
+printf "${BOLD}========================================${NC}\n"
+printf "\n"
+
+exit 0