fix: IIIF image fetch performance — retry with backoff, request size reduction, cursor warning

Three root causes of manifest analysis failures and slow performance:

1. iiif_fetcher.py: Added retry with exponential backoff (4 retries, 2s→16s)
for 429/5xx errors, global rate limiter (1 req/s), and automatic URL
rewriting from /full/full/ or /full/max/ to /full/!1500,1500/ so the
IIIF server returns pre-resized images instead of full resolution.

2. database.py: Removed cursor.close() call in SQLite PRAGMA handler —
aiosqlite returns a coroutine for close() that was never awaited,
causing RuntimeWarning at startup.

3. Tests updated for new timeout values (60s read, 15s connect) and
mocked time.sleep/time.monotonic to avoid slow tests from retry logic.

https://claude.ai/code/session_01Sxmf2zTTcjaQeZXCeXaCxr

backend/app/models/database.py

@@ -32,10 +32,13 @@ engine = create_async_engine(
 
 # Activer les clés étrangères SQLite (désactivées par défaut).
 # Nécessaire pour que ondelete="CASCADE" / "SET NULL" fonctionne.
+# Note : on n'appelle PAS cursor.close() car avec aiosqlite le curseur
+# retourne une coroutine pour close(), ce qui provoque un RuntimeWarning
+# « coroutine 'Cursor.close' was never awaited ». Le curseur PRAGMA est
+# éphémère et libéré automatiquement.
 @event.listens_for(engine.sync_engine, "connect")
 def _set_sqlite_pragma(dbapi_conn, _connection_record):
-    cursor = dbapi_conn.execute("PRAGMA foreign_keys=ON")
-    cursor.close()
+    dbapi_conn.execute("PRAGMA foreign_keys=ON")
 
 async_session_factory = async_sessionmaker(
     engine,

backend/app/services/ingest/iiif_fetcher.py

@@ -1,15 +1,21 @@
 """
 Téléchargement d'images depuis des URLs IIIF via httpx.
+
+Inclut un rate-limiter global et un retry avec backoff exponentiel
+pour respecter les limites des serveurs IIIF patrimoniaux (Gallica, etc.).
 """
 # 1. stdlib
 import logging
+import re
+import threading
+import time
 
 # 2. third-party
 import httpx
 
 logger = logging.getLogger(__name__)
 
-_DEFAULT_TIMEOUT = 30.0  # secondes (connect 10s + read 30s)
+_DEFAULT_TIMEOUT = 60.0  # secondes (connect 15s + read 60s)
 
 _HEADERS = {
     "User-Agent": (
@@ -19,10 +25,119 @@ _HEADERS = {
     "Accept": "image/jpeg,image/png,image/*,*/*",
 }
 
+# ── Rate-limiter global ────────────────────────────────────────────────────
+# Gallica and similar IIIF servers enforce strict rate limits.
+# We enforce a minimum delay between consecutive requests.
+_MIN_REQUEST_INTERVAL = 1.0  # secondes entre deux requêtes
+_rate_lock = threading.Lock()
+_last_request_time = 0.0
+
+# ── Retry configuration ───────────────────────────────────────────────────
+_MAX_RETRIES = 4
+_INITIAL_BACKOFF = 2.0  # secondes, doublé à chaque retry
+
+
+def _wait_rate_limit() -> None:
+    """Attend si nécessaire pour respecter le débit maximal vers les serveurs IIIF."""
+    global _last_request_time
+    with _rate_lock:
+        now = time.monotonic()
+        elapsed = now - _last_request_time
+        if elapsed < _MIN_REQUEST_INTERVAL:
+            time.sleep(_MIN_REQUEST_INTERVAL - elapsed)
+        _last_request_time = time.monotonic()
+
+
+def _fetch_with_retry(url: str, timeout: float) -> httpx.Response:
+    """GET avec retry et backoff exponentiel sur 429 / 5xx.
+
+    Respecte le header Retry-After si présent dans la réponse 429.
+    """
+    backoff = _INITIAL_BACKOFF
+    last_exc: Exception | None = None
+
+    for attempt in range(_MAX_RETRIES + 1):
+        _wait_rate_limit()
+        try:
+            response = httpx.get(
+                url,
+                headers=_HEADERS,
+                follow_redirects=True,
+                timeout=httpx.Timeout(timeout, connect=15.0),
+            )
+            if response.status_code == 429 or response.status_code >= 500:
+                # Respect Retry-After header if present
+                retry_after = response.headers.get("Retry-After")
+                if retry_after:
+                    try:
+                        wait_time = float(retry_after)
+                    except ValueError:
+                        wait_time = backoff
+                else:
+                    wait_time = backoff
+
+                if attempt < _MAX_RETRIES:
+                    logger.warning(
+                        "HTTP %d — retry %d/%d dans %.1fs",
+                        response.status_code,
+                        attempt + 1,
+                        _MAX_RETRIES,
+                        wait_time,
+                        extra={"url": url},
+                    )
+                    time.sleep(wait_time)
+                    backoff *= 2
+                    continue
+                # Last attempt: raise
+                response.raise_for_status()
+
+            response.raise_for_status()
+            return response
+
+        except httpx.TimeoutException as exc:
+            last_exc = exc
+            if attempt < _MAX_RETRIES:
+                logger.warning(
+                    "Timeout — retry %d/%d dans %.1fs",
+                    attempt + 1,
+                    _MAX_RETRIES,
+                    backoff,
+                    extra={"url": url},
+                )
+                time.sleep(backoff)
+                backoff *= 2
+                continue
+            raise
+
+    # Should not reach here, but just in case
+    raise last_exc or RuntimeError(f"Échec après {_MAX_RETRIES} retries : {url}")
+
+
+def _rewrite_full_to_reduced(url: str, max_px: int = 1500) -> str:
+    """Réécrit une URL IIIF /full/full/ ou /full/max/ en /full/!{max_px},{max_px}/.
+
+    Cela demande au serveur IIIF de redimensionner côté serveur au lieu de
+    retourner l'image en pleine résolution. Beaucoup plus rapide et respectueux
+    des quotas serveur.
+
+    Si l'URL n'est pas une URL IIIF standard, elle est retournée inchangée.
+    """
+    # Match IIIF Image API pattern: .../full/(full|max)/0/(default|native).(jpg|png|...)
+    pattern = r"(/full/)(full|max)(/0/)"
+    replacement = rf"\g<1>!{max_px},{max_px}\3"
+    new_url = re.sub(pattern, replacement, url)
+    if new_url != url:
+        logger.info("URL IIIF réécrite: full → !%d,%d", max_px, max_px, extra={"original": url})
+    return new_url
+
 
 def fetch_iiif_image(url: str, timeout: float = _DEFAULT_TIMEOUT) -> bytes:
     """Télécharge une image depuis une URL IIIF complète.
 
+    Si l'URL demande la pleine résolution (/full/full/ ou /full/max/),
+    elle est automatiquement réécrite pour demander un d��rivé 1500px max
+    côté serveur, ce qui est plus rapide et évite le rate-limiting.
+
     Args:
         url: URL complète de l'image (ex. https://.../full/max/0/default.jpg).
         timeout: délai maximal en secondes (défaut : 60 s).
@@ -35,14 +150,9 @@ def fetch_iiif_image(url: str, timeout: float = _DEFAULT_TIMEOUT) -> bytes:
         httpx.TimeoutException: si la requête dépasse le délai.
         httpx.RequestError: pour toute autre erreur réseau.
     """
+    url = _rewrite_full_to_reduced(url)
     logger.info("Fetching IIIF image", extra={"url": url})
-    response = httpx.get(
-        url,
-        headers=_HEADERS,
-        follow_redirects=True,
-        timeout=httpx.Timeout(timeout, connect=10.0),
-    )
-    response.raise_for_status()
+    response = _fetch_with_retry(url, timeout)
     logger.info(
         "IIIF image fetched",
         extra={"url": url, "size_bytes": len(response.content)},
@@ -71,13 +181,7 @@ def fetch_iiif_derivative(
     # Pattern IIIF Image API : !w,h = "best fit" (le serveur choisit)
     derivative_url = f"{service_url.rstrip('/')}/full/!{max_px},{max_px}/0/default.jpg"
     logger.info("Fetching IIIF derivative", extra={"url": derivative_url, "max_px": max_px})
-    response = httpx.get(
-        derivative_url,
-        headers=_HEADERS,
-        follow_redirects=True,
-        timeout=httpx.Timeout(timeout, connect=10.0),
-    )
-    response.raise_for_status()
+    response = _fetch_with_retry(derivative_url, timeout)
     logger.info(
         "IIIF derivative fetched",
         extra={"url": derivative_url, "size_bytes": len(response.content)},

backend/tests/test_image_pipeline.py

@@ -261,8 +261,11 @@ def test_fetch_iiif_image_success():
     """Retourne les bytes de l'image si la requête réussit."""
     fake_bytes = _make_jpeg_bytes(100, 100)
 
-    with patch("app.services.ingest.iiif_fetcher.httpx.get") as mock_get:
+    with patch("app.services.ingest.iiif_fetcher.httpx.get") as mock_get, \
+         patch("app.services.ingest.iiif_fetcher.time.sleep"), \
+         patch("app.services.ingest.iiif_fetcher.time.monotonic", return_value=0.0):
         mock_response = MagicMock()
+        mock_response.status_code = 200
         mock_response.content = fake_bytes
         mock_response.raise_for_status.return_value = None
         mock_get.return_value = mock_response
@@ -272,15 +275,18 @@ def test_fetch_iiif_image_success():
     assert result == fake_bytes
     _, kwargs = mock_get.call_args
     assert kwargs["follow_redirects"] is True
-    # Timeout is now an httpx.Timeout object (connect=10s, read=30s)
-    assert kwargs["timeout"].connect == 10.0
-    assert kwargs["timeout"].read == 30.0
+    # Timeout is an httpx.Timeout object (connect=15s, read=60s)
+    assert kwargs["timeout"].connect == 15.0
+    assert kwargs["timeout"].read == 60.0
 
 
 def test_fetch_iiif_image_http_error():
-    """Propage HTTPStatusError si le serveur répond 404."""
-    with patch("app.services.ingest.iiif_fetcher.httpx.get") as mock_get:
+    """Propage HTTPStatusError si le serveur répond 404 (pas de retry sur 4xx hors 429)."""
+    with patch("app.services.ingest.iiif_fetcher.httpx.get") as mock_get, \
+         patch("app.services.ingest.iiif_fetcher.time.sleep"), \
+         patch("app.services.ingest.iiif_fetcher.time.monotonic", return_value=0.0):
         mock_response = MagicMock()
+        mock_response.status_code = 404
         mock_response.raise_for_status.side_effect = httpx.HTTPStatusError(
             "404 Not Found",
             request=MagicMock(),
@@ -293,8 +299,10 @@ def test_fetch_iiif_image_http_error():
 
 
 def test_fetch_iiif_image_timeout():
-    """Propage TimeoutException si la requête dépasse le délai."""
-    with patch("app.services.ingest.iiif_fetcher.httpx.get") as mock_get:
+    """Propage TimeoutException après épuisement des retries."""
+    with patch("app.services.ingest.iiif_fetcher.httpx.get") as mock_get, \
+         patch("app.services.ingest.iiif_fetcher.time.sleep"), \
+         patch("app.services.ingest.iiif_fetcher.time.monotonic", return_value=0.0):
         mock_get.side_effect = httpx.TimeoutException("timed out")
 
         with pytest.raises(httpx.TimeoutException):
@@ -305,8 +313,11 @@ def test_fetch_iiif_image_custom_timeout():
     """Le timeout personnalisé est bien transmis à httpx.get."""
     fake_bytes = _make_jpeg_bytes(50, 50)
 
-    with patch("app.services.ingest.iiif_fetcher.httpx.get") as mock_get:
+    with patch("app.services.ingest.iiif_fetcher.httpx.get") as mock_get, \
+         patch("app.services.ingest.iiif_fetcher.time.sleep"), \
+         patch("app.services.ingest.iiif_fetcher.time.monotonic", return_value=0.0):
         mock_response = MagicMock()
+        mock_response.status_code = 200
         mock_response.content = fake_bytes
         mock_response.raise_for_status.return_value = None
         mock_get.return_value = mock_response
@@ -314,9 +325,9 @@ def test_fetch_iiif_image_custom_timeout():
         fetch_iiif_image("https://example.com/img.jpg", timeout=120.0)
 
     _, kwargs = mock_get.call_args
-    # Custom timeout wraps in httpx.Timeout(120.0, connect=10.0)
+    # Custom timeout wraps in httpx.Timeout(120.0, connect=15.0)
     assert kwargs["timeout"].read == 120.0
-    assert kwargs["timeout"].connect == 10.0
+    assert kwargs["timeout"].connect == 15.0
 
 
 # ---------------------------------------------------------------------------