Upload folder using huggingface_hub

app/core/usage_tracker.py

@@ -104,6 +104,14 @@ class UsageTracker:
     Extended to support tenant isolation: each API key is linked to a tenant.
     """
 
+    # Whether the Postgres api_keys schema check has already run in this
+    # process. Class-level, not per-instance: the schema is a property of
+    # the database, not of a tracker object, and every instance in a
+    # process points at the same DATABASE_URL. Guarded by a lock because
+    # _get_pg_conn is called from request threads.
+    _pg_schema_ready: bool = False
+    _pg_schema_lock = threading.Lock()
+
     def __init__(self, db_path: str = "arf_usage.db",
                  redis_url: Optional[str] = None,
                  pepper: Optional[str] = None):
@@ -181,30 +189,77 @@ class UsageTracker:
         Rows come back as dict-like objects (row["col"]) via RealDictCursor,
         matching the sqlite3.Row access pattern used elsewhere in this file.
 
-        Retries the initial connect with backoff -- Render's internal
-        "dpg-*" hostname has been observed unresolvable for the first
-        couple of seconds after a cold container boot (2026-08-25 incident:
-        two consecutive "could not translate host name" crashes, then a
-        clean connect on the third restart with no config change). A DNS
-        race should not require a crash loop to self-heal; other genuine
-        connection errors (bad credentials, wrong host) still raise after
-        the attempts below are exhausted, preserving fail-closed."""
+        Retries the initial connect with backoff, but how patiently depends
+        on who is asking:
+
+        - **Startup** (`warm_up`, `retries=True`): a short DNS blip during a
+          cold container boot has been observed on Render, so a few seconds
+          of retrying is worth it to come up cleanly.
+        - **Request path** (the default, `retries=False`): fails fast. A
+          request thread that blocks for 15s on a database outage doesn't
+          make the request succeed; it holds a worker thread hostage, and
+          under any concurrency the pool is exhausted and the whole service
+          stops responding -- including its health endpoint. A prompt 503 is
+          strictly better than a slow one.
+
+        Genuine connection errors (bad credentials, wrong host) still raise
+        either way, preserving fail-closed."""
         if not hasattr(self._local, "pg_conn") or self._local.pg_conn.closed:
-            last_exc: Optional[psycopg2.OperationalError] = None
-            for attempt in range(_PG_CONNECT_MAX_ATTEMPTS):
-                try:
-                    self._local.pg_conn = psycopg2.connect(
-                        self._pg_dsn, cursor_factory=psycopg2.extras.RealDictCursor)
-                    last_exc = None
-                    break
-                except psycopg2.OperationalError as exc:
-                    last_exc = exc
-                    if attempt < _PG_CONNECT_MAX_ATTEMPTS - 1:
-                        time.sleep(_PG_CONNECT_BACKOFF_BASE * (2 ** attempt))
-            if last_exc is not None:
-                raise last_exc
+            self._connect_pg(retries=False)
         yield self._local.pg_conn
 
+    def _connect_pg(self, retries: bool) -> None:
+        """Open this thread's Postgres connection, optionally retrying."""
+        attempts = _PG_CONNECT_MAX_ATTEMPTS if retries else 1
+        last_exc: Optional[psycopg2.OperationalError] = None
+        for attempt in range(attempts):
+            try:
+                self._local.pg_conn = psycopg2.connect(
+                    self._pg_dsn, cursor_factory=psycopg2.extras.RealDictCursor)
+                last_exc = None
+                break
+            except psycopg2.OperationalError as exc:
+                last_exc = exc
+                if attempt < attempts - 1:
+                    time.sleep(_PG_CONNECT_BACKOFF_BASE * (2 ** attempt))
+        if last_exc is not None:
+            raise last_exc
+        self._ensure_pg_schema(self._local.pg_conn)
+
+    def warm_up(self) -> bool:
+        """Best-effort startup connection, with retries.
+
+        Returns True if Postgres is reachable and the api_keys schema is
+        ready. Returns False -- rather than raising -- when it isn't, so a
+        caller can log the degradation and still start serving. That
+        asymmetry is the point: an unreachable database at boot should cost
+        api_keys-backed functionality, not the entire service.
+        """
+        try:
+            self._connect_pg(retries=True)
+            return True
+        except psycopg2.OperationalError:
+            return False
+
+    def _ensure_pg_schema(self, conn) -> None:
+        """Run the api_keys schema check once per process, on the first
+        connection that actually succeeds.
+
+        Guarded by a process-wide flag rather than done in __init__ so a
+        database that is unreachable at startup doesn't prevent the tracker
+        from existing -- it just means the first request that needs
+        Postgres pays for the schema check, and requests before that get a
+        clean 503 from enforce_quota instead of the whole service being
+        down. The statements are all IF NOT EXISTS, so re-running them on a
+        later process is harmless."""
+        if UsageTracker._pg_schema_ready:
+            return
+        with UsageTracker._pg_schema_lock:
+            if UsageTracker._pg_schema_ready:
+                return
+            self._create_pg_schema(conn)
+            UsageTracker._pg_schema_ready = True
+
     @staticmethod
     def _pg_execute(conn, sql: str, params: tuple = ()):
         """Run a query against a Postgres connection and return the cursor,
@@ -214,9 +269,13 @@ class UsageTracker:
         cur.execute(sql, params)
         return cur
 
-    def _init_pg_db(self):
+    def _create_pg_schema(self, conn):
         """Idempotently ensure the Postgres api_keys table/index exist.
 
+        Takes an already-open connection rather than acquiring one, because
+        its only caller is _ensure_pg_schema, which runs *from inside*
+        _get_pg_conn -- acquiring another connection here would recurse.
+
         The canonical schema is the Alembic migration
         (alembic/versions/*_create_api_keys_table.py) -- but nothing in this
         codebase runs `alembic upgrade head` automatically on deploy (a
@@ -226,33 +285,46 @@ class UsageTracker:
         already uses for its SQLite tables keeps both services (and tests)
         working whether or not the migration has actually been applied.
         Column set/types must stay in sync with that migration."""
-        with self._get_pg_conn() as conn:
-            self._pg_execute(conn, """
-                CREATE TABLE IF NOT EXISTS api_keys (
-                    id SERIAL PRIMARY KEY,
-                    tenant_id VARCHAR(64) NOT NULL,
-                    tier VARCHAR(32) NOT NULL,
-                    created_at TIMESTAMP NOT NULL,
-                    last_used_at TIMESTAMP,
-                    is_active BOOLEAN NOT NULL DEFAULT true,
-                    salt VARCHAR(64) NOT NULL,
-                    key_hash VARCHAR(64) NOT NULL,
-                    lookup_hash VARCHAR(64) NOT NULL
-                )
-            """)
-            self._pg_execute(conn, """
-                CREATE UNIQUE INDEX IF NOT EXISTS uq_api_keys_lookup_hash
-                ON api_keys (lookup_hash)
-            """)
-            self._pg_execute(conn, """
-                CREATE INDEX IF NOT EXISTS ix_api_keys_tenant_id
-                ON api_keys (tenant_id)
-            """)
-            conn.commit()
+        self._pg_execute(conn, """
+            CREATE TABLE IF NOT EXISTS api_keys (
+                id SERIAL PRIMARY KEY,
+                tenant_id VARCHAR(64) NOT NULL,
+                tier VARCHAR(32) NOT NULL,
+                created_at TIMESTAMP NOT NULL,
+                last_used_at TIMESTAMP,
+                is_active BOOLEAN NOT NULL DEFAULT true,
+                salt VARCHAR(64) NOT NULL,
+                key_hash VARCHAR(64) NOT NULL,
+                lookup_hash VARCHAR(64) NOT NULL
+            )
+        """)
+        self._pg_execute(conn, """
+            CREATE UNIQUE INDEX IF NOT EXISTS uq_api_keys_lookup_hash
+            ON api_keys (lookup_hash)
+        """)
+        self._pg_execute(conn, """
+            CREATE INDEX IF NOT EXISTS ix_api_keys_tenant_id
+            ON api_keys (tenant_id)
+        """)
+        conn.commit()
 
     def _init_db(self):
-        """Initialise SQLite tables for usage_log/monthly_counts/idempotency_keys."""
-        self._init_pg_db()
+        """Initialise SQLite tables for usage_log/monthly_counts/idempotency_keys.
+
+        Deliberately does NOT touch Postgres. The api_keys schema is
+        ensured lazily on the first successful Postgres connection instead
+        (see _ensure_pg_schema) so that constructing a UsageTracker never
+        depends on the database being reachable *at that instant*.
+
+        This is the difference between a degraded service and no service.
+        Every other subsystem in main.py's lifespan already degrades
+        gracefully when Postgres is unreachable -- the Beta-state loader
+        logs a warning and continues -- but init_tracker raised, and
+        main.py turns that into RuntimeError, killing the process. On
+        Render that produced a crash loop that outlived the port-detection
+        window, so a DNS failure lasting seconds took the whole deploy
+        down and left the previous release serving.
+        """
         with self._get_conn() as conn:
             conn.execute("""
                 CREATE TABLE IF NOT EXISTS usage_log (

app/main.py

@@ -270,8 +270,45 @@ async def lifespan(app: FastAPI):
                 db_path=os.getenv("ARF_USAGE_DB_PATH", "arf_usage.db"),
                 redis_url=os.getenv("ARF_REDIS_URL"),
             )
-            # Seed initial API keys from environment variable (for testing / demo)
+
+            # Constructing the tracker no longer touches Postgres, so warm
+            # it deliberately here: a few seconds of retrying is worth it to
+            # come up with the api_keys schema ready.
+            #
+            # A failure is logged and survived, NOT raised. Everything above
+            # in this lifespan already degrades that way -- the Beta-state
+            # loader logs a warning on the identical error and continues --
+            # and the asymmetry here is what turned a database blip into a
+            # total outage: raising crash-loops the process, which on Render
+            # exhausts the port-detection window, fails the deploy, and
+            # leaves the previous release serving. Starting degraded means
+            # the health endpoint answers, the deploy succeeds, and API
+            # requests get a clean 503 from enforce_quota until the database
+            # is reachable -- at which point they recover with no redeploy.
+            postgres_ready = tracker.warm_up()
+            if not postgres_ready:
+                logger.error(
+                    "Usage tracker started WITHOUT a Postgres connection: api_keys "
+                    "is unreachable, so API-key validation and quota enforcement "
+                    "will fail (503) until it recovers. Check that DATABASE_URL's "
+                    "host resolves from this service, and that the database is "
+                    "running and in the same region."
+                )
+
+            # Seed initial API keys from environment variable (for testing
+            # / demo). Skipped entirely when Postgres is unreachable: every
+            # get_or_create_api_key below is a write to api_keys, so
+            # attempting it would raise straight back into the handler that
+            # kills the process -- reintroducing the crash loop the warm-up
+            # above exists to prevent.
             api_keys_json = os.getenv("ARF_API_KEYS", "{}")
+            if not postgres_ready and api_keys_json not in ("", "{}"):
+                logger.warning(
+                    "Skipping ARF_API_KEYS seeding: Postgres is unreachable. "
+                    "Seeded keys will not exist until the database recovers "
+                    "and the service is restarted."
+                )
+                api_keys_json = "{}"
             try:
                 api_keys = json.loads(api_keys_json)
                 for key, tier_str in api_keys.items():
@@ -302,8 +339,16 @@ async def lifespan(app: FastAPI):
                     "ARF_API_KEYS environment variable is not valid JSON; skipping seeding."
                 )
             app.state.usage_tracker = tracker
-            logger.info("✅ Usage tracker ready.")
+            if postgres_ready:
+                logger.info("✅ Usage tracker ready.")
+            else:
+                logger.warning("⚠️  Usage tracker started in degraded mode (no Postgres).")
         except Exception as e:
+            # Still fail closed on genuine configuration errors -- a missing
+            # or too-short ARF_KEY_PEPPER, an unset DATABASE_URL. Those never
+            # self-resolve, and starting without them would mean serving with
+            # broken API-key hashing. Database *reachability* is handled
+            # above and no longer reaches here.
             logger.critical(f"Failed to initialise usage tracker: {e}")
             raise RuntimeError("Usage tracker initialisation failed") from e
     else: