Spaces:
Running
Running
maintenance mode + vector store temp db warning
Browse files- app/api/server.py +21 -0
- app/api/v1/system.py +32 -0
- app/api/v1/vector_stores.py +0 -3
- deploy.py +29 -3
app/api/server.py
CHANGED
|
@@ -17,6 +17,7 @@ from app.core.vector_store.deps import init_vector_store_db
|
|
| 17 |
from app.services.embeddings_service import EmbeddingService
|
| 18 |
from app.services.vector_store_service import VectorStoreService
|
| 19 |
from app.api.v1.router import api_v1_router
|
|
|
|
| 20 |
|
| 21 |
_logger = get_logger(__name__)
|
| 22 |
_settings = get_settings()
|
|
@@ -102,6 +103,26 @@ def create_application() -> FastAPI:
|
|
| 102 |
allow_headers=["*"],
|
| 103 |
)
|
| 104 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 105 |
app.include_router(api_v1_router, prefix="/api/v1")
|
| 106 |
|
| 107 |
@app.get("/", include_in_schema=False)
|
|
|
|
| 17 |
from app.services.embeddings_service import EmbeddingService
|
| 18 |
from app.services.vector_store_service import VectorStoreService
|
| 19 |
from app.api.v1.router import api_v1_router
|
| 20 |
+
from app.api.v1.system import is_maintenance
|
| 21 |
|
| 22 |
_logger = get_logger(__name__)
|
| 23 |
_settings = get_settings()
|
|
|
|
| 103 |
allow_headers=["*"],
|
| 104 |
)
|
| 105 |
|
| 106 |
+
@app.middleware("http")
|
| 107 |
+
async def maintenance_middleware(request: Request, call_next):
|
| 108 |
+
if is_maintenance():
|
| 109 |
+
method = request.method
|
| 110 |
+
path = request.url.path
|
| 111 |
+
if method not in ("GET", "HEAD", "OPTIONS"):
|
| 112 |
+
if not path.startswith("/api/v1/system/maintenance"):
|
| 113 |
+
if method == "POST" and path.startswith("/api/v1/backup"):
|
| 114 |
+
pass # allow backup/restore during maintenance
|
| 115 |
+
else:
|
| 116 |
+
from starlette.responses import JSONResponse
|
| 117 |
+
return JSONResponse(
|
| 118 |
+
status_code=503,
|
| 119 |
+
content={
|
| 120 |
+
"success": False,
|
| 121 |
+
"detail": "Service is under maintenance. No write operations allowed.",
|
| 122 |
+
},
|
| 123 |
+
)
|
| 124 |
+
return await call_next(request)
|
| 125 |
+
|
| 126 |
app.include_router(api_v1_router, prefix="/api/v1")
|
| 127 |
|
| 128 |
@app.get("/", include_in_schema=False)
|
app/api/v1/system.py
CHANGED
|
@@ -36,6 +36,13 @@ _settings = get_settings()
|
|
| 36 |
_START_TIME = time.time()
|
| 37 |
_logger = get_logger(__name__)
|
| 38 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 39 |
|
| 40 |
@router.get("/health", response_model=HealthResponse, summary="Health check")
|
| 41 |
async def health(
|
|
@@ -185,3 +192,28 @@ async def upload_and_restore(
|
|
| 185 |
"files_restored": restored,
|
| 186 |
"data_dir": str(data_dir),
|
| 187 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 36 |
_START_TIME = time.time()
|
| 37 |
_logger = get_logger(__name__)
|
| 38 |
|
| 39 |
+
_MAINTENANCE_MODE: bool = False
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def is_maintenance() -> bool:
|
| 44 |
+
return _MAINTENANCE_MODE or os.environ.get("APP_UNDER_MAINTENANCE", "").lower() in ("1", "true", "yes")
|
| 45 |
+
|
| 46 |
|
| 47 |
@router.get("/health", response_model=HealthResponse, summary="Health check")
|
| 48 |
async def health(
|
|
|
|
| 192 |
"files_restored": restored,
|
| 193 |
"data_dir": str(data_dir),
|
| 194 |
}
|
| 195 |
+
|
| 196 |
+
|
| 197 |
+
@router.post("/maintenance/enable", summary="Enable maintenance mode (blocks write operations)")
|
| 198 |
+
async def enable_maintenance(token: str = Depends(require_auth)):
|
| 199 |
+
global _MAINTENANCE_MODE
|
| 200 |
+
_MAINTENANCE_MODE = True
|
| 201 |
+
_logger.warning("Maintenance mode ENABLED — all write operations blocked")
|
| 202 |
+
return {"success": True, "message": "Maintenance mode enabled", "maintenance": True}
|
| 203 |
+
|
| 204 |
+
|
| 205 |
+
@router.post("/maintenance/disable", summary="Disable maintenance mode")
|
| 206 |
+
async def disable_maintenance(token: str = Depends(require_auth)):
|
| 207 |
+
global _MAINTENANCE_MODE
|
| 208 |
+
_MAINTENANCE_MODE = False
|
| 209 |
+
_logger.warning("Maintenance mode DISABLED — write operations resumed")
|
| 210 |
+
return {"success": True, "message": "Maintenance mode disabled", "maintenance": False}
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
@router.get("/maintenance", summary="Check maintenance mode status")
|
| 214 |
+
async def maintenance_status(token: str = Depends(require_auth)):
|
| 215 |
+
return {
|
| 216 |
+
"success": True,
|
| 217 |
+
"maintenance": is_maintenance(),
|
| 218 |
+
"source": "api_flag" if _MAINTENANCE_MODE else ("env_var" if os.environ.get("APP_UNDER_MAINTENANCE") else "off"),
|
| 219 |
+
}
|
app/api/v1/vector_stores.py
CHANGED
|
@@ -101,7 +101,6 @@ async def list_vector_stores(
|
|
| 101 |
document_count=stats.get("document_count", 0),
|
| 102 |
created_at=r.created_at,
|
| 103 |
metadata=r.metadata,
|
| 104 |
-
warning=get_temp_db_warning(),
|
| 105 |
))
|
| 106 |
return VectorStoreListResponse(success=True, total=len(stores), stores=stores)
|
| 107 |
|
|
@@ -130,7 +129,6 @@ async def get_vector_store(
|
|
| 130 |
document_count=stats["document_count"],
|
| 131 |
created_at=stats["created_at"],
|
| 132 |
metadata=stats["metadata_json"],
|
| 133 |
-
warning=get_temp_db_warning(),
|
| 134 |
)
|
| 135 |
|
| 136 |
|
|
@@ -158,7 +156,6 @@ async def delete_vector_store(
|
|
| 158 |
document_count=0,
|
| 159 |
embedding_dimension=0,
|
| 160 |
created_at=record.created_at,
|
| 161 |
-
warning=get_temp_db_warning(),
|
| 162 |
)
|
| 163 |
|
| 164 |
|
|
|
|
| 101 |
document_count=stats.get("document_count", 0),
|
| 102 |
created_at=r.created_at,
|
| 103 |
metadata=r.metadata,
|
|
|
|
| 104 |
))
|
| 105 |
return VectorStoreListResponse(success=True, total=len(stores), stores=stores)
|
| 106 |
|
|
|
|
| 129 |
document_count=stats["document_count"],
|
| 130 |
created_at=stats["created_at"],
|
| 131 |
metadata=stats["metadata_json"],
|
|
|
|
| 132 |
)
|
| 133 |
|
| 134 |
|
|
|
|
| 156 |
document_count=0,
|
| 157 |
embedding_dimension=0,
|
| 158 |
created_at=record.created_at,
|
|
|
|
| 159 |
)
|
| 160 |
|
| 161 |
|
deploy.py
CHANGED
|
@@ -256,6 +256,19 @@ def http_get_json(
|
|
| 256 |
return None
|
| 257 |
|
| 258 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 259 |
def http_head(url: str, timeout: int = 30) -> Optional[int]:
|
| 260 |
try:
|
| 261 |
req = urllib.request.Request(url, method="HEAD")
|
|
@@ -562,15 +575,27 @@ def run_deployment(args: argparse.Namespace, env: dict[str, str]) -> int:
|
|
| 562 |
}
|
| 563 |
|
| 564 |
# ──────────────────────────────────────────────
|
| 565 |
-
# PHASE 1:
|
| 566 |
# ──────────────────────────────────────────────
|
| 567 |
-
print(f"\n{_bold('PHASE 1/4:
|
| 568 |
print(f" {_INFO_ICON} Backing up data from: {space_url}")
|
| 569 |
|
| 570 |
if skip_backup:
|
| 571 |
print(f" {_WARN_ICON} Backup skipped (--skip-backup)")
|
| 572 |
audit.set("backup_status", "skipped")
|
| 573 |
else:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 574 |
backup_file = persistence_dir / BACKUP_ARCHIVE
|
| 575 |
backup_url = _build_url(space_url, BACKUP_ENDPOINT)
|
| 576 |
audit.set("backup_started", _timestamp())
|
|
@@ -636,7 +661,8 @@ def run_deployment(args: argparse.Namespace, env: dict[str, str]) -> int:
|
|
| 636 |
audit.set("backup_size", total_bytes)
|
| 637 |
audit.set("backup_status", "passed")
|
| 638 |
print(f" {_OK_ICON} Files: {file_count} | Size: {_human_size(total_bytes)}")
|
| 639 |
-
|
|
|
|
| 640 |
|
| 641 |
# ──────────────────────────────────────────────
|
| 642 |
# PHASE 2: GIT COMMIT & PUSH
|
|
|
|
| 256 |
return None
|
| 257 |
|
| 258 |
|
| 259 |
+
def http_post_json(
|
| 260 |
+
url: str, headers: dict[str, str], data: Optional[bytes] = None, timeout: int = 30
|
| 261 |
+
) -> Optional[int]:
|
| 262 |
+
try:
|
| 263 |
+
req = urllib.request.Request(url, data=data, headers=headers, method="POST")
|
| 264 |
+
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
| 265 |
+
return resp.status
|
| 266 |
+
except urllib.error.HTTPError as e:
|
| 267 |
+
return e.code
|
| 268 |
+
except Exception:
|
| 269 |
+
return None
|
| 270 |
+
|
| 271 |
+
|
| 272 |
def http_head(url: str, timeout: int = 30) -> Optional[int]:
|
| 273 |
try:
|
| 274 |
req = urllib.request.Request(url, method="HEAD")
|
|
|
|
| 575 |
}
|
| 576 |
|
| 577 |
# ──────────────────────────────────────────────
|
| 578 |
+
# PHASE 1: LOCK WRITES + BACKUP
|
| 579 |
# ──────────────────────────────────────────────
|
| 580 |
+
print(f"\n{_bold('PHASE 1/4: Lock Writes & Backup')}")
|
| 581 |
print(f" {_INFO_ICON} Backing up data from: {space_url}")
|
| 582 |
|
| 583 |
if skip_backup:
|
| 584 |
print(f" {_WARN_ICON} Backup skipped (--skip-backup)")
|
| 585 |
audit.set("backup_status", "skipped")
|
| 586 |
else:
|
| 587 |
+
# Enable maintenance mode to freeze data state
|
| 588 |
+
maint_url = _build_url(space_url, "/api/v1/system/maintenance/enable")
|
| 589 |
+
maint_enabled = False
|
| 590 |
+
try:
|
| 591 |
+
status = http_post_json(maint_url, auth_headers, timeout=15)
|
| 592 |
+
if status and status == 200:
|
| 593 |
+
maint_enabled = True
|
| 594 |
+
print(f" {_OK_ICON} Maintenance enabled — writes blocked")
|
| 595 |
+
except Exception:
|
| 596 |
+
print(f" {_WARN_ICON} Could not enable maintenance (endpoint may not exist yet)")
|
| 597 |
+
print(f" {_WARN_ICON} Data may be lost if writes occur during deployment")
|
| 598 |
+
|
| 599 |
backup_file = persistence_dir / BACKUP_ARCHIVE
|
| 600 |
backup_url = _build_url(space_url, BACKUP_ENDPOINT)
|
| 601 |
audit.set("backup_started", _timestamp())
|
|
|
|
| 661 |
audit.set("backup_size", total_bytes)
|
| 662 |
audit.set("backup_status", "passed")
|
| 663 |
print(f" {_OK_ICON} Files: {file_count} | Size: {_human_size(total_bytes)}")
|
| 664 |
+
if maint_enabled:
|
| 665 |
+
print(f" {_OK_ICON} Writes frozen — backup is consistent")
|
| 666 |
|
| 667 |
# ──────────────────────────────────────────────
|
| 668 |
# PHASE 2: GIT COMMIT & PUSH
|