telemetry relay
Browse files- Dockerfile +10 -0
- README.md +13 -3
- app.py +92 -0
- requirements.txt +3 -0
Dockerfile
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
FROM python:3.11-slim
|
| 2 |
+
RUN useradd -m -u 1000 user
|
| 3 |
+
WORKDIR /app
|
| 4 |
+
COPY requirements.txt .
|
| 5 |
+
RUN pip install --no-cache-dir -r requirements.txt
|
| 6 |
+
COPY app.py .
|
| 7 |
+
USER user
|
| 8 |
+
ENV HOME=/home/user
|
| 9 |
+
EXPOSE 7860
|
| 10 |
+
CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]
|
README.md
CHANGED
|
@@ -1,10 +1,20 @@
|
|
| 1 |
---
|
| 2 |
title: ReverseBass Beta Relay
|
| 3 |
-
emoji:
|
| 4 |
colorFrom: green
|
| 5 |
-
colorTo:
|
| 6 |
sdk: docker
|
|
|
|
| 7 |
pinned: false
|
| 8 |
---
|
| 9 |
|
| 10 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
title: ReverseBass Beta Relay
|
| 3 |
+
emoji: 📡
|
| 4 |
colorFrom: green
|
| 5 |
+
colorTo: gray
|
| 6 |
sdk: docker
|
| 7 |
+
app_port: 7860
|
| 8 |
pinned: false
|
| 9 |
---
|
| 10 |
|
| 11 |
+
# SA3 Sampler beta telemetry relay
|
| 12 |
+
|
| 13 |
+
Receives the performance log the SA3 Sampler beta uploads automatically and stores it in a
|
| 14 |
+
private dataset repository. The plugin carries no credential; this service holds the write
|
| 15 |
+
token as a Space secret (`HF_TOKEN`).
|
| 16 |
+
|
| 17 |
+
- `GET /` health and counters
|
| 18 |
+
- `POST /logs/<install-id>/<YYYYMMDD>` body: the day's JSON-lines log (max 1 MB)
|
| 19 |
+
|
| 20 |
+
The log contains hardware facts, timings and errors only. See the beta privacy note.
|
app.py
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""SA3 Sampler beta telemetry relay.
|
| 2 |
+
|
| 3 |
+
The plugin POSTs its daily performance log here without any credential. This service holds
|
| 4 |
+
the Hugging Face token (Space secret HF_TOKEN) and writes the file into the private dataset
|
| 5 |
+
repo. Nothing else is exposed: one route, one file per (installation id, day), size-capped,
|
| 6 |
+
rate-limited per client address.
|
| 7 |
+
"""
|
| 8 |
+
import os
|
| 9 |
+
import re
|
| 10 |
+
import threading
|
| 11 |
+
import time
|
| 12 |
+
|
| 13 |
+
from fastapi import FastAPI, Request
|
| 14 |
+
from fastapi.responses import JSONResponse
|
| 15 |
+
from huggingface_hub import HfApi
|
| 16 |
+
|
| 17 |
+
REPO = os.environ.get("TELEMETRY_REPO", "VortexSamples/ReverseBass-Beta-Telemetry")
|
| 18 |
+
TOKEN = os.environ.get("HF_TOKEN", "")
|
| 19 |
+
MAX_BYTES = 1_000_000 # a day of events is a few KB; anything bigger is not ours
|
| 20 |
+
PER_HOUR = 120 # uploads per client address per hour
|
| 21 |
+
ID_RE = re.compile(r"^[0-9a-zA-Z-]{6,64}$")
|
| 22 |
+
DAY_RE = re.compile(r"^\d{8}$")
|
| 23 |
+
|
| 24 |
+
app = FastAPI(docs_url=None, redoc_url=None, openapi_url=None)
|
| 25 |
+
api = HfApi(token=TOKEN or None)
|
| 26 |
+
commit_lock = threading.Lock()
|
| 27 |
+
recent: dict[str, list[float]] = {}
|
| 28 |
+
recent_lock = threading.Lock()
|
| 29 |
+
stats = {"accepted": 0, "rejected": 0, "started": int(time.time())}
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def client_ip(request: Request) -> str:
|
| 33 |
+
fwd = request.headers.get("x-forwarded-for", "")
|
| 34 |
+
return (fwd.split(",")[0].strip() if fwd else (request.client.host if request.client else "?"))
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def rate_limited(ip: str) -> bool:
|
| 38 |
+
now = time.time()
|
| 39 |
+
with recent_lock:
|
| 40 |
+
hits = [t for t in recent.get(ip, []) if now - t < 3600]
|
| 41 |
+
if len(hits) >= PER_HOUR:
|
| 42 |
+
recent[ip] = hits
|
| 43 |
+
return True
|
| 44 |
+
hits.append(now)
|
| 45 |
+
recent[ip] = hits
|
| 46 |
+
if len(recent) > 10000: # keep the table bounded
|
| 47 |
+
for k in [k for k, v in recent.items() if not v or now - v[-1] > 3600]:
|
| 48 |
+
recent.pop(k, None)
|
| 49 |
+
return False
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def reject(status: int, why: str):
|
| 53 |
+
stats["rejected"] += 1
|
| 54 |
+
return JSONResponse({"ok": False, "why": why}, status_code=status)
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
@app.get("/")
|
| 58 |
+
def root():
|
| 59 |
+
return {"ok": True, "service": "sa3-telemetry-relay", "repo": REPO, **stats}
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
@app.post("/logs/{install_id}/{day}")
|
| 63 |
+
async def put_log(install_id: str, day: str, request: Request):
|
| 64 |
+
if not TOKEN:
|
| 65 |
+
return reject(503, "relay has no token configured")
|
| 66 |
+
if not ID_RE.match(install_id) or not DAY_RE.match(day):
|
| 67 |
+
return reject(400, "bad id or day")
|
| 68 |
+
if rate_limited(client_ip(request)):
|
| 69 |
+
return reject(429, "too many uploads")
|
| 70 |
+
body = await request.body()
|
| 71 |
+
if not body or len(body) > MAX_BYTES:
|
| 72 |
+
return reject(413 if body else 400, "bad size")
|
| 73 |
+
try:
|
| 74 |
+
text = body.decode("utf-8")
|
| 75 |
+
except UnicodeDecodeError:
|
| 76 |
+
return reject(400, "not utf-8")
|
| 77 |
+
if not text.lstrip().startswith("{"):
|
| 78 |
+
return reject(400, "not a log")
|
| 79 |
+
|
| 80 |
+
path = f"logs/{install_id}/{day}.jsonl"
|
| 81 |
+
try:
|
| 82 |
+
with commit_lock:
|
| 83 |
+
api.upload_file(path_or_fileobj=body, path_in_repo=path, repo_id=REPO, repo_type="dataset",
|
| 84 |
+
commit_message=f"telemetry {install_id[:8]} {day}")
|
| 85 |
+
except Exception as e: # noqa: BLE001
|
| 86 |
+
msg = str(e)
|
| 87 |
+
if "No files have been modified" in msg or "no changes" in msg.lower():
|
| 88 |
+
stats["accepted"] += 1
|
| 89 |
+
return {"ok": True, "bytes": len(body), "unchanged": True}
|
| 90 |
+
return reject(502, "upload failed: " + msg[:200])
|
| 91 |
+
stats["accepted"] += 1
|
| 92 |
+
return {"ok": True, "bytes": len(body)}
|
requirements.txt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
fastapi==0.115.6
|
| 2 |
+
uvicorn==0.34.0
|
| 3 |
+
huggingface_hub==0.36.0
|