Create app.py

app.py

@@ -0,0 +1,53 @@
+from fastapi import FastAPI, Request
+from fastapi.responses import StreamingResponse, FileResponse
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.staticfiles import StaticFiles
+import onnxruntime as ort
+import numpy as np
+import tiktoken
+import json
+import os
+
+app = FastAPI()
+
+# CORS erlauben
+app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"])
+
+# Modell & Tokenizer laden
+tokenizer = tiktoken.get_encoding("gpt2")
+MODEL_PATH = "SmaLLMPro_350M_int8.onnx"
+# Nutzt optimierte CPU-Einstellungen
+session = ort.InferenceSession(MODEL_PATH, providers=['CPUExecutionProvider'])
+
+def top_k_sample(logits, k=50, temp=0.7):
+    logits = logits / temp
+    # Numpy ist VIEL schneller als JS-Schleifen
+    top_k_indices = np.argpartition(logits, -k)[-k:]
+    top_k_logits = logits[top_k_indices]
+    exp_logits = np.exp(top_k_logits - np.max(top_k_logits))
+    probs = exp_logits / np.sum(exp_logits)
+    return int(np.random.choice(top_k_indices, p=probs))
+
+@app.post("/chat")
+async def chat(request: Request):
+    data = await request.json()
+    prompt = f"Instruction:\n{data['prompt']}\n\nResponse:\n"
+    tokens = tokenizer.encode(prompt)
+    
+    async def generate():
+        nonlocal tokens
+        for _ in range(data.get('maxLen', 100)):
+            ctx = tokens[-1024:]
+            padded = np.zeros((1, 1024), dtype=np.int64)
+            padded[0, -len(ctx):] = ctx
+            
+            outputs = session.run(None, {'input': padded})
+            logits = outputs[0][0, -1, :50304]
+            
+            next_token = top_k_sample(logits, k=data.get('topK', 25), temp=data.get('temp', 0.5))
+            if next_token == 50256: break
+            
+            tokens.append(next_token)
+            yield f"data: {json.dumps({'token': tokenizer.decode([next_token])})}\n\n"
+
+    return StreamingResponse(generate(), media_type="text/event-stream")