Update server.py

server.py

@@ -33,7 +33,7 @@ MODEL_ID   = "gemma-4-e2b"
 model_status = "loading"
 engine       = None
 _engine_ctx  = None
-engine_lock  = threading.Lock()
+engine_lock  = threading.BoundedSemaphore(value=2)
 
 app = Flask(__name__)
 CORS(app)
@@ -108,8 +108,11 @@ def parse_openai_messages(messages: list) -> tuple[str, bytes | None]:
 def _run_real_model_generator(ask: str, image_bytes: bytes | None):
     """Yields text chunks as they are generated by the model."""
     import litert_lm
-    # engine_lock ensures only 1 request processes at a time to prevent RAM crashes
-    with engine_lock:
+    # engine_lock ensures only 2 requests process at a time to prevent RAM crashes
+    if not engine_lock.acquire(timeout=30):
+        raise RuntimeError("Server busy. Try again shortly.")
+
+    try:
         with engine.create_conversation() as conv:
             if image_bytes:
                 msg = litert_lm.Contents.of(
@@ -125,6 +128,8 @@ def _run_real_model_generator(ask: str, image_bytes: bytes | None):
                         text = part.get("text", "")
                         if text:
                             yield text
+    finally:
+        engine_lock.release()
 
 
 def _run_mock_generator(ask: str, has_image: bool):
@@ -236,4 +241,9 @@ if __name__ == "__main__":
     port = int(os.environ.get("PORT", 5173))
     threading.Thread(target=load_model, daemon=True).start()
     print(f"[INFO] Gemma OpenAI-Compatible API listening on :{port}", flush=True)
-    app.run(host="0.0.0.0", port=port, debug=False)
+    app.run(
+        host="0.0.0.0",
+        port=port,
+        debug=False,
+        threaded=True,
+    )