Update main.py

main.py

@@ -1,9 +1,9 @@
 from fastapi import FastAPI, HTTPException
 from fastapi.responses import JSONResponse
+import asyncio
 import time
 import psutil
 import os
-import asyncio
 
 app = FastAPI()
 
@@ -11,9 +11,10 @@ process = psutil.Process(os.getpid())
 last_cpu_times = process.cpu_times()
 last_time = time.time()
 
-# Limit concurrent requests
-MAX_CONCURRENT_REQUESTS = 20
-semaphore = asyncio.Semaphore(MAX_CONCURRENT_REQUESTS)
+# Queue settings
+MAX_QUEUE_SIZE = 100       # Max number of requests waiting in the queue
+NUM_WORKERS = 20           # Number of concurrent workers
+request_queue = asyncio.Queue(MAX_QUEUE_SIZE)
 
 def get_cpu_percent():
     global last_cpu_times, last_time
@@ -31,19 +32,35 @@ def simulate_work():
         total += i
     return total
 
-async def limited_worker():
-    async with semaphore:
-        # Actual work happens here
-        simulate_work()
-        return {"ok": True, "time": int(time.time() * 1000)}
+async def worker():
+    while True:
+        fut = await request_queue.get()
+        try:
+            result = simulate_work()
+            fut.set_result({"ok": True, "time": int(time.time() * 1000)})
+        except Exception as e:
+            fut.set_exception(e)
+        finally:
+            request_queue.task_done()
+
+# Start worker tasks on startup
+@app.on_event("startup")
+async def startup_event():
+    for _ in range(NUM_WORKERS):
+        asyncio.create_task(worker())
 
 @app.get("/")
 async def root():
-    try:
-        result = await limited_worker()
-        return result
-    except Exception as e:
-        raise HTTPException(status_code=500, detail=str(e))
+    if request_queue.full():
+        # Queue is full, reject request
+        raise HTTPException(status_code=429, detail="Server busy, try again later")
+    
+    # Create a future to get the result from the worker
+    loop = asyncio.get_event_loop()
+    fut = loop.create_future()
+    await request_queue.put(fut)
+    result = await fut
+    return result
 
 @app.get("/metrics")
 async def metrics():