Initial commit: LFM2-700M FastAPI inference app

Dockerfile

@@ -0,0 +1,50 @@
+FROM python:3.11-slim
+
+# HuggingFace Spaces runs as a non-root user
+RUN useradd -m -u 1000 user
+WORKDIR /app
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        git \
+    && rm -rf /var/lib/apt/lists/*
+
+# Python deps
+COPY --chown=user requirements.txt ./
+RUN pip install --no-cache-dir -r requirements.txt
+
+# App source
+COPY --chown=user . .
+
+# ---------------------------------------------------------------------------
+# Pre-download model into the image at build time so startup is instant.
+# Pass --build-arg HF_TOKEN=<your_token> to authenticate during build.
+# On HuggingFace Spaces, set HF_TOKEN as a Space secret.
+# ---------------------------------------------------------------------------
+ARG HF_TOKEN=""
+ENV MODEL_ID=gsstec/LFM2-700M
+ENV HF_HOME=/app/model_cache
+ENV TRANSFORMERS_CACHE=/app/model_cache
+ENV HF_HUB_DISABLE_PROGRESS_BARS=1
+
+RUN HF_TOKEN="${HF_TOKEN}" \
+    HUGGING_FACE_HUB_TOKEN="${HF_TOKEN}" \
+    python - <<'EOF'
+from transformers import AutoModelForCausalLM, AutoTokenizer
+import os
+model_id = os.environ["MODEL_ID"]
+token = os.environ.get("HF_TOKEN") or None
+print(f"Pre-downloading {model_id} (authenticated={bool(token)}) ...")
+AutoTokenizer.from_pretrained(model_id, token=token, clean_up_tokenization_spaces=False)
+AutoModelForCausalLM.from_pretrained(model_id, token=token, device_map="cpu")
+print("Done.")
+EOF
+
+RUN chown -R user:user /app/model_cache
+
+USER user
+
+EXPOSE 7860
+
+# python app.py — model loads at module level, then uvicorn starts in the same process
+# No forking, no workers — model is guaranteed loaded before first request
+CMD ["python", "app.py"]

README.md

@@ -0,0 +1,71 @@
+---
+title: LFM2-700M
+emoji: 🤖
+colorFrom: blue
+colorTo: purple
+sdk: docker
+pinned: false
+app_port: 7860
+---
+
+# LFM2-700M — FastAPI Inference Space
+
+Runs [gsstec/LFM2-700M](https://huggingface.co/gsstec/LFM2-700M) behind a lightweight **FastAPI** server.
+
+## Endpoints
+
+| Method | Path | Description |
+|--------|------|-------------|
+| `GET`  | `/`  | Interactive HTML docs page |
+| `GET`  | `/health` | Liveness check |
+| `POST` | `/chat`   | Generate a response |
+
+## POST /chat — request body
+
+```json
+{
+  "messages": [
+    {"role": "user", "content": "What is C. elegans?"}
+  ],
+  "max_new_tokens": 512,
+  "temperature": 0.3,
+  "min_p": 0.15,
+  "repetition_penalty": 1.05,
+  "stream": false
+}
+```
+
+Set `"stream": true` to receive a streaming `text/plain` response via chunked transfer.
+
+## POST /chat — response body (non-streaming)
+
+```json
+{
+  "response": "C. elegans, also known as Caenorhabditis elegans, is a small ..."
+}
+```
+
+## Local development
+
+```bash
+pip install -r requirements.txt
+uvicorn app:app --reload --port 7860
+```
+
+## Docker
+
+```bash
+docker build -t lfm2-700m .
+docker run -p 7860:7860 lfm2-700m
+```
+
+## Deploy to HuggingFace Spaces
+
+1. Create a new Space at https://huggingface.co/new-space → choose **Docker** SDK
+2. Push all files to the Space repo:
+   ```bash
+   git remote add space https://huggingface.co/spaces/<your-org>/<space-name>
+   git push space main
+   ```
+
+> **GPU tip:** Upgrade the Space hardware to a GPU tier and uncomment `attn_implementation="flash_attention_2"` in `app.py` for faster inference.

app.py

@@ -0,0 +1,202 @@
+import os
+import json
+import torch
+import logging
+
+# ---------------------------------------------------------------------------
+# Environment must be configured BEFORE importing transformers/huggingface_hub
+# ---------------------------------------------------------------------------
+os.environ.setdefault("HF_HOME", "/app/model_cache")
+os.environ.setdefault("TRANSFORMERS_CACHE", "/app/model_cache")
+
+# Propagate HF_TOKEN so authenticated requests are made automatically
+_hf_token = os.getenv("HF_TOKEN")
+if _hf_token:
+    os.environ["HUGGING_FACE_HUB_TOKEN"] = _hf_token
+    os.environ["HF_TOKEN"] = _hf_token
+
+from fastapi import FastAPI, HTTPException, Request
+from fastapi.exceptions import RequestValidationError
+from fastapi.responses import HTMLResponse, StreamingResponse, JSONResponse
+from pydantic import BaseModel
+from typing import List
+from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
+from threading import Thread
+
+logging.basicConfig(level=logging.WARNING)
+logger = logging.getLogger(__name__)
+logging.getLogger("uvicorn.access").setLevel(logging.INFO)
+
+MODEL_ID = os.getenv("MODEL_ID", "gsstec/LFM2-700M")
+
+# Max out CPU threads for faster matrix ops
+_num_threads = int(os.getenv("NUM_THREADS", os.cpu_count() or 4))
+torch.set_num_threads(_num_threads)
+torch.set_num_interop_threads(max(1, _num_threads // 2))
+
+# ---------------------------------------------------------------------------
+# Model loading — runs at import time, before uvicorn accepts any requests
+# ---------------------------------------------------------------------------
+_token_kwarg = {"token": _hf_token} if _hf_token else {}
+print(f"Loading model: {MODEL_ID}  (CPU threads: {_num_threads}, authenticated={bool(_hf_token)})")
+
+tokenizer = AutoTokenizer.from_pretrained(
+    MODEL_ID,
+    clean_up_tokenization_spaces=False,
+    **_token_kwarg,
+)
+tokenizer.padding_side = "left"
+if tokenizer.pad_token is None:
+    tokenizer.pad_token = tokenizer.eos_token
+
+model = AutoModelForCausalLM.from_pretrained(
+    MODEL_ID,
+    device_map="auto",
+    torch_dtype=torch.bfloat16,
+    # attn_implementation="flash_attention_2",  # uncomment on compatible GPU
+    **_token_kwarg,
+)
+model.eval()
+
+print("Model ready.")
+
+# ---------------------------------------------------------------------------
+# App
+# ---------------------------------------------------------------------------
+app = FastAPI(title="LFM2-700M Inference API")
+
+
+# ---------------------------------------------------------------------------
+# Validation error handler
+# ---------------------------------------------------------------------------
+@app.exception_handler(RequestValidationError)
+async def validation_exception_handler(request: Request, exc: RequestValidationError):
+    body = await request.body()
+    safe_errors = [
+        {k: (v.decode(errors="replace") if isinstance(v, bytes) else v)
+         for k, v in err.items()}
+        for err in exc.errors()
+    ]
+    logger.error(f"422 {request.method} {request.url} | {safe_errors}")
+    return JSONResponse(
+        status_code=422,
+        content={"detail": safe_errors, "body_received": body.decode(errors="replace")},
+    )
+
+
+# ---------------------------------------------------------------------------
+# Schemas
+# ---------------------------------------------------------------------------
+class Message(BaseModel):
+    role: str       # "user" | "assistant" | "system"
+    content: str
+
+
+class ChatRequest(BaseModel):
+    messages: List[Message]
+    max_new_tokens: int = 512
+    temperature: float = 0.3
+    min_p: float = 0.15
+    repetition_penalty: float = 1.05
+    stream: bool = False
+
+
+class ChatResponse(BaseModel):
+    response: str
+
+
+class AskRequest(BaseModel):
+    question: str
+    max_new_tokens: int = 512
+    temperature: float = 0.3
+    min_p: float = 0.15
+    repetition_penalty: float = 1.05
+
+
+# ---------------------------------------------------------------------------
+# Shared helpers
+# ---------------------------------------------------------------------------
+def build_input_ids(messages: List[Message]) -> torch.Tensor:
+    chat = [{"role": m.role, "content": m.content} for m in messages]
+    encoding = tokenizer.apply_chat_template(
+        chat,
+        add_generation_prompt=True,
+        return_tensors="pt",
+        tokenize=True,
+        return_dict=True,
+    )
+    return encoding["input_ids"].to(model.device)
+
+
+def _generate(input_ids: torch.Tensor, req, streamer=None) -> torch.Tensor:
+    kwargs = dict(
+        input_ids=input_ids,
+        do_sample=True,
+        temperature=req.temperature,
+        min_p=req.min_p,
+        repetition_penalty=req.repetition_penalty,
+        max_new_tokens=req.max_new_tokens,
+        use_cache=True,
+    )
+    if streamer:
+        kwargs["streamer"] = streamer
+    with torch.inference_mode():
+        return model.generate(**kwargs)
+
+
+# ---------------------------------------------------------------------------
+# Routes
+# ---------------------------------------------------------------------------
+_UI_HTML = open(os.path.join(os.path.dirname(__file__), "static", "index.html"), encoding="utf-8").read()
+
+
+@app.get("/", response_class=HTMLResponse)
+@app.get("/ui", response_class=HTMLResponse)
+def ui():
+    """Chat UI — served at both / and /ui"""
+    return _UI_HTML
+
+
+@app.get("/health")
+def health():
+    return {"status": "ok", "model": MODEL_ID, "ready": True,
+            "cpu_threads": _num_threads}
+
+
+@app.post("/chat")
+async def chat(request: Request):
+    try:
+        body = await request.body()
+        req = ChatRequest(**json.loads(body))
+    except Exception as e:
+        raise HTTPException(status_code=422, detail=f"Invalid request body: {e}")
+
+    input_ids = build_input_ids(req.messages)
+
+    if req.stream:
+        streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
+        Thread(target=_generate, args=(input_ids, req, streamer), daemon=True).start()
+        return StreamingResponse((tok for tok in streamer), media_type="text/plain")
+
+    output = _generate(input_ids, req)
+    new_tokens = output[0][input_ids.shape[-1]:]
+    return ChatResponse(response=tokenizer.decode(new_tokens, skip_special_tokens=True))
+
+
+@app.post("/ask")
+async def ask(request: Request):
+    try:
+        body = await request.body()
+        req = AskRequest(**json.loads(body))
+    except Exception as e:
+        raise HTTPException(status_code=422, detail=f"Invalid request body: {e}")
+
+    input_ids = build_input_ids([Message(role="user", content=req.question)])
+    output = _generate(input_ids, req)
+    new_tokens = output[0][input_ids.shape[-1]:]
+    return ChatResponse(response=tokenizer.decode(new_tokens, skip_special_tokens=True))
+
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=7860, log_level="warning", access_log=True)

hf_space_clone

@@ -0,0 +1 @@
+Subproject commit 1ca5003f8c3bf11b0547164f7e7fc9cf2c2d3a20

requirements.txt

@@ -0,0 +1,7 @@
+fastapi>=0.111.0
+uvicorn[standard]>=0.29.0
+transformers>=4.51.0
+torch>=2.3.0
+accelerate>=0.30.0
+pydantic>=2.0.0
+sentencepiece

start.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+# start.sh — pre-warm model then hand off to uvicorn
+# Python loads app.py (model loading) then exec's uvicorn so it inherits the process
+
+exec python -c "
+import app          # runs all module-level code: model loads here
+import uvicorn
+uvicorn.run(app.app, host='0.0.0.0', port=7860, log_level='warning', access_log=True)
+"

static/index.html

@@ -0,0 +1,228 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8" />
+  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+  <title>LFM2-700M Chat</title>
+  <style>
+    *, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
+
+    body {
+      font-family: -apple-system, "Segoe UI", system-ui, sans-serif;
+      background: #f7f8fa;
+      color: #1f2328;
+      height: 100vh;
+      display: flex;
+      flex-direction: column;
+      align-items: center;
+      padding: 24px 16px;
+    }
+
+    h1 {
+      font-size: 1.2rem;
+      font-weight: 600;
+      margin-bottom: 16px;
+      color: #1f2328;
+    }
+    h1 span { color: #3b82d4; }
+
+    #chat-box {
+      width: 100%;
+      max-width: 720px;
+      flex: 1;
+      overflow-y: auto;
+      background: #fff;
+      border: 1px solid #e5e7eb;
+      border-radius: 10px;
+      padding: 16px;
+      display: flex;
+      flex-direction: column;
+      gap: 12px;
+      margin-bottom: 16px;
+    }
+
+    .msg {
+      max-width: 85%;
+      padding: 10px 14px;
+      border-radius: 10px;
+      line-height: 1.6;
+      font-size: 0.93rem;
+      white-space: pre-wrap;
+      word-break: break-word;
+    }
+    .msg.user {
+      align-self: flex-end;
+      background: #3b82d4;
+      color: #fff;
+      border-bottom-right-radius: 2px;
+    }
+    .msg.assistant {
+      align-self: flex-start;
+      background: #f0f2f5;
+      color: #1f2328;
+      border-bottom-left-radius: 2px;
+    }
+    .msg.thinking {
+      align-self: flex-start;
+      background: #f0f2f5;
+      color: #57606a;
+      font-style: italic;
+      animation: pulse 1.2s ease-in-out infinite;
+    }
+    @keyframes pulse { 0%,100%{opacity:1} 50%{opacity:0.4} }
+
+    #input-row {
+      display: flex;
+      width: 100%;
+      max-width: 720px;
+      gap: 8px;
+    }
+
+    #question {
+      flex: 1;
+      padding: 10px 14px;
+      font-size: 0.95rem;
+      border: 1px solid #e5e7eb;
+      border-radius: 8px;
+      outline: none;
+      background: #fff;
+      resize: none;
+      height: 44px;
+      line-height: 1.4;
+      font-family: inherit;
+    }
+    #question:focus { border-color: #3b82d4; }
+
+    #send-btn {
+      padding: 0 20px;
+      height: 44px;
+      background: #3b82d4;
+      color: #fff;
+      border: none;
+      border-radius: 8px;
+      font-size: 0.95rem;
+      font-weight: 500;
+      cursor: pointer;
+      white-space: nowrap;
+    }
+    #send-btn:disabled { background: #a5c4ef; cursor: not-allowed; }
+    #send-btn:hover:not(:disabled) { background: #2b6cbf; }
+
+    #settings {
+      display: flex;
+      gap: 12px;
+      width: 100%;
+      max-width: 720px;
+      margin-bottom: 10px;
+      flex-wrap: wrap;
+    }
+    .setting {
+      display: flex;
+      align-items: center;
+      gap: 6px;
+      font-size: 0.82rem;
+      color: #57606a;
+    }
+    .setting input[type=range] { width: 80px; }
+    .setting span { min-width: 28px; text-align: right; font-weight: 500; color: #1f2328; }
+  </style>
+</head>
+<body>
+
+  <h1>🤖 <span>LFM2-700M</span> Chat</h1>
+
+  <div id="settings">
+    <div class="setting">
+      Max tokens
+      <input type="range" id="max_tokens" min="64" max="1024" step="64" value="512"
+             oninput="document.getElementById('mt_val').textContent=this.value">
+      <span id="mt_val">512</span>
+    </div>
+    <div class="setting">
+      Temperature
+      <input type="range" id="temperature" min="0.01" max="1.5" step="0.01" value="0.3"
+             oninput="document.getElementById('temp_val').textContent=parseFloat(this.value).toFixed(2)">
+      <span id="temp_val">0.30</span>
+    </div>
+    <div class="setting">
+      Min-P
+      <input type="range" id="min_p" min="0" max="1" step="0.01" value="0.15"
+             oninput="document.getElementById('mp_val').textContent=parseFloat(this.value).toFixed(2)">
+      <span id="mp_val">0.15</span>
+    </div>
+  </div>
+
+  <div id="chat-box"></div>
+
+  <div id="input-row">
+    <textarea id="question" placeholder="Ask anything…" rows="1"></textarea>
+    <button id="send-btn" onclick="sendQuestion()">Send</button>
+  </div>
+
+  <script>
+    const API = '';   // same origin — served by FastAPI
+
+    const chatBox  = document.getElementById('chat-box');
+    const input    = document.getElementById('question');
+    const sendBtn  = document.getElementById('send-btn');
+
+    // Send on Enter (Shift+Enter = newline)
+    input.addEventListener('keydown', e => {
+      if (e.key === 'Enter' && !e.shiftKey) { e.preventDefault(); sendQuestion(); }
+    });
+
+    function appendMsg(role, text) {
+      const div = document.createElement('div');
+      div.className = `msg ${role}`;
+      div.textContent = text;
+      chatBox.appendChild(div);
+      chatBox.scrollTop = chatBox.scrollHeight;
+      return div;
+    }
+
+    async function sendQuestion() {
+      const q = input.value.trim();
+      if (!q) return;
+
+      input.value = '';
+      sendBtn.disabled = true;
+      appendMsg('user', q);
+      const thinking = appendMsg('thinking', 'Thinking…');
+
+      const body = {
+        question:           q,
+        max_new_tokens:     parseInt(document.getElementById('max_tokens').value),
+        temperature:        parseFloat(document.getElementById('temperature').value),
+        min_p:              parseFloat(document.getElementById('min_p').value),
+        repetition_penalty: 1.05
+      };
+
+      try {
+        const res = await fetch(`${API}/ask`, {
+          method:  'POST',
+          headers: { 'Content-Type': 'application/json' },
+          body:    JSON.stringify(body)
+        });
+
+        if (!res.ok) {
+          const err = await res.text();
+          thinking.className = 'msg assistant';
+          thinking.textContent = `Error ${res.status}: ${err}`;
+        } else {
+          const data = await res.json();
+          thinking.className  = 'msg assistant';
+          thinking.textContent = data.response;
+        }
+      } catch (err) {
+        thinking.className  = 'msg assistant';
+        thinking.textContent = `Network error: ${err.message}`;
+      }
+
+      sendBtn.disabled = false;
+      input.focus();
+      chatBox.scrollTop = chatBox.scrollHeight;
+    }
+  </script>
+
+</body>
+</html>