Spaces:
Running
Running
Update app/app.py
Browse files- app/app.py +70 -87
app/app.py
CHANGED
|
@@ -11,35 +11,38 @@
|
|
| 11 |
# All fundament services are injected via the `fundaments` dictionary.
|
| 12 |
# Direct execution is blocked by design.
|
| 13 |
#
|
| 14 |
-
# SANDBOX
|
| 15 |
-
#
|
| 16 |
-
#
|
| 17 |
-
#
|
|
|
|
|
|
|
| 18 |
# =============================================================================
|
| 19 |
|
| 20 |
-
from quart import Quart, request, jsonify # async Flask β required for async
|
| 21 |
import logging
|
| 22 |
-
from waitress import serve # WSGI server β keeps
|
| 23 |
import threading # bank-pattern: each blocking service gets its own thread
|
| 24 |
import requests # sync HTTP for health check worker
|
| 25 |
import time
|
| 26 |
from datetime import datetime
|
| 27 |
import asyncio
|
| 28 |
-
import sys
|
| 29 |
from typing import Dict, Any, Optional
|
| 30 |
|
| 31 |
# =============================================================================
|
| 32 |
# Import app/* modules
|
| 33 |
-
#
|
|
|
|
| 34 |
# =============================================================================
|
| 35 |
from . import mcp # MCP transport layer (stdio / SSE)
|
| 36 |
-
from . import providers # API provider registry (LLM, Search, Web)
|
| 37 |
-
from . import models # Model config + token/rate limits
|
| 38 |
-
from . import tools # MCP tool definitions + provider mapping
|
| 39 |
-
from . import db_sync # Internal SQLite IPC
|
| 40 |
-
# db_sync β
|
| 41 |
-
|
| 42 |
-
|
|
|
|
| 43 |
# from . import discord_api # Discord bot integration
|
| 44 |
# from . import hf_hooks # HuggingFace Space hooks
|
| 45 |
# from . import git_hooks # GitHub/GitLab webhook handler
|
|
@@ -48,54 +51,20 @@ from . import db_sync # Internal SQLite IPC β app/* state & communication
|
|
| 48 |
# =============================================================================
|
| 49 |
# Loggers β one per module for clean log filtering
|
| 50 |
# =============================================================================
|
| 51 |
-
logger
|
| 52 |
-
#
|
| 53 |
-
#
|
| 54 |
-
# logger_tools = logging.getLogger('tools')
|
| 55 |
# logger_providers = logging.getLogger('providers')
|
| 56 |
-
# logger_models
|
| 57 |
-
# logger_db_sync
|
|
|
|
| 58 |
|
| 59 |
# =============================================================================
|
| 60 |
-
#
|
| 61 |
# =============================================================================
|
| 62 |
app = Quart(__name__)
|
| 63 |
START_TIME = datetime.utcnow()
|
| 64 |
|
| 65 |
-
# =============================================================================
|
| 66 |
-
# Global service references (set during initialize_services)
|
| 67 |
-
# =============================================================================
|
| 68 |
-
_fundaments: Optional[Dict[str, Any]] = None
|
| 69 |
-
PORT = None
|
| 70 |
-
|
| 71 |
-
# =============================================================================
|
| 72 |
-
# Service initialization
|
| 73 |
-
# =============================================================================
|
| 74 |
-
def initialize_services(fundaments: Dict[str, Any]) -> None:
|
| 75 |
-
"""
|
| 76 |
-
Initializes all app/* services with injected fundaments from Guardian.
|
| 77 |
-
Called once during start_application β sets global service references.
|
| 78 |
-
"""
|
| 79 |
-
global _fundaments, PORT
|
| 80 |
-
|
| 81 |
-
_fundaments = fundaments
|
| 82 |
-
PORT = fundaments["config"].get_int("PORT", 7860)
|
| 83 |
-
|
| 84 |
-
# Initialize internal SQLite state store for app/* IPC
|
| 85 |
-
db_sync.initialize()
|
| 86 |
-
|
| 87 |
-
# Initialize provider registry from app/.pyfun + ENV key presence check
|
| 88 |
-
providers.initialize(fundaments["config"])
|
| 89 |
-
|
| 90 |
-
# Initialize model registry from app/.pyfun
|
| 91 |
-
models.initialize()
|
| 92 |
-
|
| 93 |
-
# Initialize tool registry β tools only register if their provider is active
|
| 94 |
-
tools.initialize(providers, models, fundaments)
|
| 95 |
-
|
| 96 |
-
logger.info("app/* services initialized.")
|
| 97 |
-
|
| 98 |
-
|
| 99 |
# =============================================================================
|
| 100 |
# Background workers
|
| 101 |
# =============================================================================
|
|
@@ -103,31 +72,33 @@ def start_mcp_in_thread() -> None:
|
|
| 103 |
"""
|
| 104 |
Starts the MCP Hub (stdio or SSE) in its own thread with its own event loop.
|
| 105 |
Mirrors the bank-thread pattern from the Discord bot architecture.
|
|
|
|
| 106 |
"""
|
| 107 |
loop = asyncio.new_event_loop()
|
| 108 |
asyncio.set_event_loop(loop)
|
| 109 |
try:
|
| 110 |
-
loop.run_until_complete(mcp.start_mcp(
|
| 111 |
finally:
|
| 112 |
loop.close()
|
| 113 |
|
| 114 |
|
| 115 |
-
def health_check_worker() -> None:
|
| 116 |
"""
|
| 117 |
Periodic self-ping to keep the app alive on hosting platforms (e.g. HuggingFace).
|
| 118 |
Runs in its own daemon thread β does not block the main loop.
|
|
|
|
| 119 |
"""
|
| 120 |
while True:
|
| 121 |
time.sleep(3600)
|
| 122 |
try:
|
| 123 |
-
response = requests.get(f"http://127.0.0.1:{
|
| 124 |
logger.info(f"Health check ping: {response.status_code}")
|
| 125 |
except Exception as e:
|
| 126 |
logger.error(f"Health check failed: {e}")
|
| 127 |
|
| 128 |
|
| 129 |
# =============================================================================
|
| 130 |
-
#
|
| 131 |
# =============================================================================
|
| 132 |
|
| 133 |
@app.route("/", methods=["GET"])
|
|
@@ -141,7 +112,7 @@ async def health_check():
|
|
| 141 |
"status": "running",
|
| 142 |
"service": "Universal MCP Hub",
|
| 143 |
"uptime_seconds": int(uptime.total_seconds()),
|
| 144 |
-
"active_providers": providers.get_active_names()
|
| 145 |
})
|
| 146 |
|
| 147 |
|
|
@@ -161,14 +132,9 @@ async def api_endpoint():
|
|
| 161 |
async def crypto_endpoint():
|
| 162 |
"""
|
| 163 |
Encrypted API endpoint.
|
| 164 |
-
|
| 165 |
-
Only active if encryption_service is available in fundaments.
|
| 166 |
"""
|
| 167 |
-
|
| 168 |
-
if not encryption_service:
|
| 169 |
-
return jsonify({"error": "Encryption service not available"}), 503
|
| 170 |
-
|
| 171 |
-
# TODO: decrypt payload, dispatch, re-encrypt response
|
| 172 |
data = await request.get_json()
|
| 173 |
return jsonify({"status": "not_implemented"}), 501
|
| 174 |
|
|
@@ -191,7 +157,7 @@ async def crypto_endpoint():
|
|
| 191 |
|
| 192 |
|
| 193 |
# =============================================================================
|
| 194 |
-
# Main entry point β called by Guardian (main.py)
|
| 195 |
# =============================================================================
|
| 196 |
async def start_application(fundaments: Dict[str, Any]) -> None:
|
| 197 |
"""
|
|
@@ -200,20 +166,21 @@ async def start_application(fundaments: Dict[str, Any]) -> None:
|
|
| 200 |
|
| 201 |
Args:
|
| 202 |
fundaments: Dictionary of initialized services from Guardian (main.py).
|
| 203 |
-
|
|
|
|
| 204 |
"""
|
| 205 |
logger.info("Application starting...")
|
| 206 |
|
| 207 |
-
#
|
| 208 |
-
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
#
|
| 216 |
-
|
| 217 |
|
| 218 |
# --- Log active fundament services ---
|
| 219 |
if encryption_service:
|
|
@@ -231,25 +198,41 @@ async def start_application(fundaments: Dict[str, Any]) -> None:
|
|
| 231 |
if not db_service:
|
| 232 |
logger.info("Database-free mode active (e.g. Discord bot, API client).")
|
| 233 |
|
| 234 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 235 |
mcp_thread = threading.Thread(target=start_mcp_in_thread, daemon=True)
|
| 236 |
mcp_thread.start()
|
| 237 |
logger.info("MCP Hub thread started.")
|
| 238 |
|
| 239 |
-
# Allow MCP to initialize before Flask comes up
|
| 240 |
await asyncio.sleep(1)
|
| 241 |
|
| 242 |
# --- Start health check worker ---
|
| 243 |
-
health_thread = threading.Thread(
|
|
|
|
|
|
|
|
|
|
|
|
|
| 244 |
health_thread.start()
|
| 245 |
|
| 246 |
-
# --- Start
|
| 247 |
def run_server():
|
| 248 |
-
serve(app, host="0.0.0.0", port=
|
| 249 |
|
| 250 |
server_thread = threading.Thread(target=run_server, daemon=True)
|
| 251 |
server_thread.start()
|
| 252 |
-
logger.info(f"HTTP server started on port {
|
| 253 |
|
| 254 |
logger.info("All services running. Entering heartbeat loop...")
|
| 255 |
|
|
@@ -278,4 +261,4 @@ if __name__ == '__main__':
|
|
| 278 |
"security": None,
|
| 279 |
}
|
| 280 |
|
| 281 |
-
asyncio.run(start_application(test_fundaments))
|
|
|
|
| 11 |
# All fundament services are injected via the `fundaments` dictionary.
|
| 12 |
# Direct execution is blocked by design.
|
| 13 |
#
|
| 14 |
+
# SANDBOX RULES:
|
| 15 |
+
# - fundaments dict is ONLY unpacked inside start_application()
|
| 16 |
+
# - fundaments are NEVER stored globally or passed to other app/* modules
|
| 17 |
+
# - app/* modules read their own config from app/.pyfun
|
| 18 |
+
# - app/* internal state/IPC uses app/db_sync.py (SQLite) β NOT postgresql.py
|
| 19 |
+
# - Secrets stay in .env β Guardian reads them β never touched by app/*
|
| 20 |
# =============================================================================
|
| 21 |
|
| 22 |
+
from quart import Quart, request, jsonify # async Flask β required for async providers + Neon DB
|
| 23 |
import logging
|
| 24 |
+
from waitress import serve # WSGI server β keeps HTTP non-blocking alongside asyncio
|
| 25 |
import threading # bank-pattern: each blocking service gets its own thread
|
| 26 |
import requests # sync HTTP for health check worker
|
| 27 |
import time
|
| 28 |
from datetime import datetime
|
| 29 |
import asyncio
|
|
|
|
| 30 |
from typing import Dict, Any, Optional
|
| 31 |
|
| 32 |
# =============================================================================
|
| 33 |
# Import app/* modules
|
| 34 |
+
# Each module reads its own config from app/.pyfun independently.
|
| 35 |
+
# NO fundaments passed into these modules!
|
| 36 |
# =============================================================================
|
| 37 |
from . import mcp # MCP transport layer (stdio / SSE)
|
| 38 |
+
from . import providers # API provider registry (LLM, Search, Web) β reads app/.pyfun
|
| 39 |
+
from . import models # Model config + token/rate limits β reads app/.pyfun
|
| 40 |
+
from . import tools # MCP tool definitions + provider mapping β reads app/.pyfun
|
| 41 |
+
from . import db_sync # Internal SQLite IPC for app/* state & communication
|
| 42 |
+
# db_sync β postgresql.py! Cloud DB is Guardian-only.
|
| 43 |
+
from . import config as app_config # app/.pyfun parser β used only in app/*
|
| 44 |
+
|
| 45 |
+
# Future modules (uncomment when ready):
|
| 46 |
# from . import discord_api # Discord bot integration
|
| 47 |
# from . import hf_hooks # HuggingFace Space hooks
|
| 48 |
# from . import git_hooks # GitHub/GitLab webhook handler
|
|
|
|
| 51 |
# =============================================================================
|
| 52 |
# Loggers β one per module for clean log filtering
|
| 53 |
# =============================================================================
|
| 54 |
+
logger = logging.getLogger('application')
|
| 55 |
+
# logger_mcp = logging.getLogger('mcp')
|
| 56 |
+
# logger_tools = logging.getLogger('tools')
|
|
|
|
| 57 |
# logger_providers = logging.getLogger('providers')
|
| 58 |
+
# logger_models = logging.getLogger('models')
|
| 59 |
+
# logger_db_sync = logging.getLogger('db_sync')
|
| 60 |
+
# logger_config = logging.getLogger('config')
|
| 61 |
|
| 62 |
# =============================================================================
|
| 63 |
+
# Quart app instance
|
| 64 |
# =============================================================================
|
| 65 |
app = Quart(__name__)
|
| 66 |
START_TIME = datetime.utcnow()
|
| 67 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 68 |
# =============================================================================
|
| 69 |
# Background workers
|
| 70 |
# =============================================================================
|
|
|
|
| 72 |
"""
|
| 73 |
Starts the MCP Hub (stdio or SSE) in its own thread with its own event loop.
|
| 74 |
Mirrors the bank-thread pattern from the Discord bot architecture.
|
| 75 |
+
mcp.py reads its own config from app/.pyfun β no fundaments passed in.
|
| 76 |
"""
|
| 77 |
loop = asyncio.new_event_loop()
|
| 78 |
asyncio.set_event_loop(loop)
|
| 79 |
try:
|
| 80 |
+
loop.run_until_complete(mcp.start_mcp())
|
| 81 |
finally:
|
| 82 |
loop.close()
|
| 83 |
|
| 84 |
|
| 85 |
+
def health_check_worker(port: int) -> None:
|
| 86 |
"""
|
| 87 |
Periodic self-ping to keep the app alive on hosting platforms (e.g. HuggingFace).
|
| 88 |
Runs in its own daemon thread β does not block the main loop.
|
| 89 |
+
Port passed directly β no global state needed.
|
| 90 |
"""
|
| 91 |
while True:
|
| 92 |
time.sleep(3600)
|
| 93 |
try:
|
| 94 |
+
response = requests.get(f"http://127.0.0.1:{port}/")
|
| 95 |
logger.info(f"Health check ping: {response.status_code}")
|
| 96 |
except Exception as e:
|
| 97 |
logger.error(f"Health check failed: {e}")
|
| 98 |
|
| 99 |
|
| 100 |
# =============================================================================
|
| 101 |
+
# Quart Routes
|
| 102 |
# =============================================================================
|
| 103 |
|
| 104 |
@app.route("/", methods=["GET"])
|
|
|
|
| 112 |
"status": "running",
|
| 113 |
"service": "Universal MCP Hub",
|
| 114 |
"uptime_seconds": int(uptime.total_seconds()),
|
| 115 |
+
"active_providers": providers.get_active_names(),
|
| 116 |
})
|
| 117 |
|
| 118 |
|
|
|
|
| 132 |
async def crypto_endpoint():
|
| 133 |
"""
|
| 134 |
Encrypted API endpoint.
|
| 135 |
+
Encryption handled by app/* layer β no direct fundaments access here.
|
|
|
|
| 136 |
"""
|
| 137 |
+
# TODO: implement via app/* encryption wrapper
|
|
|
|
|
|
|
|
|
|
|
|
|
| 138 |
data = await request.get_json()
|
| 139 |
return jsonify({"status": "not_implemented"}), 501
|
| 140 |
|
|
|
|
| 157 |
|
| 158 |
|
| 159 |
# =============================================================================
|
| 160 |
+
# Main entry point β called exclusively by Guardian (main.py)
|
| 161 |
# =============================================================================
|
| 162 |
async def start_application(fundaments: Dict[str, Any]) -> None:
|
| 163 |
"""
|
|
|
|
| 166 |
|
| 167 |
Args:
|
| 168 |
fundaments: Dictionary of initialized services from Guardian (main.py).
|
| 169 |
+
Services are unpacked here and NEVER stored globally or
|
| 170 |
+
passed into other app/* modules.
|
| 171 |
"""
|
| 172 |
logger.info("Application starting...")
|
| 173 |
|
| 174 |
+
# =========================================================================
|
| 175 |
+
# Unpack fundaments β ONLY here, NEVER elsewhere in app/*
|
| 176 |
+
# These are the 6 fundament services from fundaments/*
|
| 177 |
+
# =========================================================================
|
| 178 |
+
config_service = fundaments["config"] # fundaments/config_handler.py
|
| 179 |
+
db_service = fundaments["db"] # fundaments/postgresql.py β None if not configured
|
| 180 |
+
encryption_service = fundaments["encryption"] # fundaments/encryption.py β None if keys not set
|
| 181 |
+
access_control_service = fundaments["access_control"] # fundaments/access_control.py β None if no DB
|
| 182 |
+
user_handler_service = fundaments["user_handler"] # fundaments/user_handler.py β None if no DB
|
| 183 |
+
security_service = fundaments["security"] # fundaments/security.py β None if deps missing
|
| 184 |
|
| 185 |
# --- Log active fundament services ---
|
| 186 |
if encryption_service:
|
|
|
|
| 198 |
if not db_service:
|
| 199 |
logger.info("Database-free mode active (e.g. Discord bot, API client).")
|
| 200 |
|
| 201 |
+
# =========================================================================
|
| 202 |
+
# Initialize app/* internal services
|
| 203 |
+
# Each module reads app/.pyfun independently via app/config.py
|
| 204 |
+
# NO fundaments passed in here!
|
| 205 |
+
# =========================================================================
|
| 206 |
+
db_sync.initialize() # SQLite IPC store for app/* β unrelated to postgresql.py
|
| 207 |
+
providers.initialize() # reads app/.pyfun [LLM_PROVIDERS] [SEARCH_PROVIDERS]
|
| 208 |
+
models.initialize() # reads app/.pyfun [MODELS]
|
| 209 |
+
tools.initialize() # reads app/.pyfun [TOOLS]
|
| 210 |
+
|
| 211 |
+
# --- Read PORT from app/.pyfun [HUB] ---
|
| 212 |
+
port = int(app_config.get_hub().get("HUB_PORT", "7860"))
|
| 213 |
+
|
| 214 |
+
# --- Start MCP Hub in its own thread ---
|
| 215 |
mcp_thread = threading.Thread(target=start_mcp_in_thread, daemon=True)
|
| 216 |
mcp_thread.start()
|
| 217 |
logger.info("MCP Hub thread started.")
|
| 218 |
|
|
|
|
| 219 |
await asyncio.sleep(1)
|
| 220 |
|
| 221 |
# --- Start health check worker ---
|
| 222 |
+
health_thread = threading.Thread(
|
| 223 |
+
target=health_check_worker,
|
| 224 |
+
args=(port,),
|
| 225 |
+
daemon=True
|
| 226 |
+
)
|
| 227 |
health_thread.start()
|
| 228 |
|
| 229 |
+
# --- Start Quart via Waitress in its own thread ---
|
| 230 |
def run_server():
|
| 231 |
+
serve(app, host="0.0.0.0", port=port)
|
| 232 |
|
| 233 |
server_thread = threading.Thread(target=run_server, daemon=True)
|
| 234 |
server_thread.start()
|
| 235 |
+
logger.info(f"HTTP server started on port {port}.")
|
| 236 |
|
| 237 |
logger.info("All services running. Entering heartbeat loop...")
|
| 238 |
|
|
|
|
| 261 |
"security": None,
|
| 262 |
}
|
| 263 |
|
| 264 |
+
asyncio.run(start_application(test_fundaments))
|