Support 2GB GeoTIFF library uploads with streamed writes and progress UI

DEPLOYMENT.md

@@ -132,6 +132,9 @@ Set these in each Space’s **Settings → Repository secrets / Variables** if n
 
 | Variable | Purpose |
 |----------|---------|
+| `APP_MODE` | Set to `dda` on **satdetect-dev** only (enables DDA library UI) |
+| `MAX_GEOTIFF_MB` | Library GeoTIFF upload cap (default **2048** = 2 GB on dev) |
+| `MAX_IMAGE_MB` | PNG/JPEG library cap (default 50 MB) |
 | `SECRET_KEY` | Optional legacy JWT setting (login disabled) |
 | `DATABASE_URL` | PostgreSQL instead of SQLite (optional) |
 | `SMTP_USER` / `SMTP_PASS` | Email notifications via Gmail SMTP |

Dockerfile

@@ -21,7 +21,7 @@ WORKDIR /app
 
 # Build-time info + cache-bust:
 # Changing APP_BUILD forces Docker to re-run subsequent layers (including pip install).
-ARG APP_BUILD=25
+ARG APP_BUILD=26
 ENV APP_BUILD=${APP_BUILD}
 ENV GDAL_CONFIG=/usr/bin/gdal-config
 RUN echo "Docker build start: APP_BUILD=${APP_BUILD}" && python -V
@@ -51,4 +51,4 @@ ENV PORT=7860
 EXPOSE 7860
 
 # Bind to runtime PORT so health checks always reach the server.
-CMD ["sh", "-c", "uvicorn app.main:app --host 0.0.0.0 --port ${PORT:-7860} --log-level info"]
+CMD ["sh", "-c", "uvicorn app.main:app --host 0.0.0.0 --port ${PORT:-7860} --log-level info --timeout-keep-alive 600"]

app/dda/config.py

@@ -10,8 +10,11 @@ LIBRARY_DIR = DATA_DIR / "library"
 THUMBS_DIR = LIBRARY_DIR / "thumbs"
 PREVIEWS_DIR = LIBRARY_DIR / "previews"
 
-# GeoTIFF upload limit (DDA responsible for suitable resolution per SOW)
-MAX_GEOTIFF_BYTES = int(os.environ.get("MAX_GEOTIFF_MB", "500")) * 1024 * 1024
+# GeoTIFF library upload limit (default 2 GB; override with MAX_GEOTIFF_MB on HF dev Space)
+MAX_GEOTIFF_BYTES = int(os.environ.get("MAX_GEOTIFF_MB", "2048")) * 1024 * 1024
+
+# Raster sidecar formats (PNG/JPEG) — smaller cap for library uploads
+MAX_IMAGE_BYTES = int(os.environ.get("MAX_IMAGE_MB", "50")) * 1024 * 1024
 
 ALLOWED_EXTENSIONS = {".tif", ".tiff", ".png", ".jpg", ".jpeg"}
 
@@ -24,6 +27,12 @@ def ensure_library_dirs() -> None:
             pass
 
 
+def max_upload_bytes_for_extension(ext: str) -> int:
+    if ext in (".tif", ".tiff"):
+        return MAX_GEOTIFF_BYTES
+    return MAX_IMAGE_BYTES
+
+
 def geotiff_io_available() -> bool:
     try:
         import rasterio  # noqa: F401

app/dda/geotiff_io.py

@@ -74,16 +74,24 @@ def inspect_image(path: Path) -> IngestResult:
 
 
 def raster_to_preview_png(src_path: Path, dest_path: Path, max_side: int = 512) -> None:
-    """Create RGB thumbnail/preview from GeoTIFF or raster image."""
+    """Create RGB thumbnail/preview — uses decimated read for large GeoTIFFs."""
     ext = src_path.suffix.lower()
     if ext in (".tif", ".tiff"):
         try:
             import numpy as np
             import rasterio
+            from rasterio.enums import Resampling
 
             with rasterio.open(src_path) as src:
                 count = min(3, src.count)
-                data = src.read(indexes=list(range(1, count + 1)))
+                scale = min(1.0, max_side / max(src.width, src.height, 1))
+                out_h = max(1, int(src.height * scale))
+                out_w = max(1, int(src.width * scale))
+                data = src.read(
+                    indexes=list(range(1, count + 1)),
+                    out_shape=(count, out_h, out_w),
+                    resampling=Resampling.bilinear,
+                )
                 if count == 1:
                     rgb = np.stack([data[0], data[0], data[0]])
                 else:
@@ -93,7 +101,8 @@ def raster_to_preview_png(src_path: Path, dest_path: Path, max_side: int = 512)
                     lo, hi = np.percentile(rgb, (2, 98))
                     rgb = np.clip((rgb - lo) / max(hi - lo, 1e-6), 0, 1) * 255
                 img = Image.fromarray(rgb.astype("uint8"), mode="RGB")
-                img.thumbnail((max_side, max_side), Image.Resampling.LANCZOS)
+                if max(img.size) > max_side:
+                    img.thumbnail((max_side, max_side), Image.Resampling.LANCZOS)
                 dest_path.parent.mkdir(parents=True, exist_ok=True)
                 img.save(dest_path, format="PNG")
                 return

app/dda/library_routes.py

@@ -17,14 +17,15 @@ from .config import (
     ALLOWED_EXTENSIONS,
     IS_DDA_MODE,
     LIBRARY_DIR,
-    MAX_GEOTIFF_BYTES,
     PREVIEWS_DIR,
     THUMBS_DIR,
     ensure_library_dirs,
     geotiff_io_available,
+    max_upload_bytes_for_extension,
 )
 from .geotiff_io import bounds_to_json, inspect_image, raster_to_preview_png
 from .models import DdaVillage, DdaZone, ImageAsset
+from .upload_io import stream_upload_to_file
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
@@ -64,9 +65,14 @@ def _image_to_dict(asset: ImageAsset, zone_name: str = "", village_name: str = "
 @router.get("/config")
 def dda_config():
     _require_dda()
+    from .config import MAX_GEOTIFF_BYTES, MAX_IMAGE_BYTES
+    max_gb = MAX_GEOTIFF_BYTES / (1024 ** 3)
     return {
         "mode": "dda",
         "maxUploadMb": MAX_GEOTIFF_BYTES // (1024 * 1024),
+        "maxUploadGb": round(max_gb, 2),
+        "maxGeotiffMb": MAX_GEOTIFF_BYTES // (1024 * 1024),
+        "maxImageMb": MAX_IMAGE_BYTES // (1024 * 1024),
         "geotiffEnabled": geotiff_io_available(),
         "allowedExtensions": sorted(ALLOWED_EXTENSIONS),
         "hierarchyMode": "admin",
@@ -201,15 +207,6 @@ async def upload_image(
             detail=f"Unsupported format. Allowed: {', '.join(sorted(ALLOWED_EXTENSIONS))}",
         )
 
-    raw = await file.read()
-    if not raw:
-        raise HTTPException(status_code=400, detail="File is empty")
-    if len(raw) > MAX_GEOTIFF_BYTES:
-        raise HTTPException(
-            status_code=400,
-            detail=f"File too large (max {MAX_GEOTIFF_BYTES // (1024 * 1024)} MB)",
-        )
-
     try:
         cap_date = date.fromisoformat(capture_date.strip())
     except ValueError:
@@ -221,8 +218,8 @@ async def upload_image(
     asset_uuid = uuid.uuid4().hex
     stored_name = f"{asset_uuid}{ext}"
     dest_file = LIBRARY_DIR / str(year) / stored_name
-    dest_file.parent.mkdir(parents=True, exist_ok=True)
-    dest_file.write_bytes(raw)
+    max_bytes = max_upload_bytes_for_extension(ext)
+    file_size = await stream_upload_to_file(file, dest_file, max_bytes)
 
     ingest = inspect_image(dest_file)
     has_georef = ingest.has_georef
@@ -277,7 +274,7 @@ async def upload_image(
         manual_location_json=manual_location,
         width=ingest.width,
         height=ingest.height,
-        file_size_bytes=len(raw),
+        file_size_bytes=file_size,
         uploaded_by=user.id,
     )
     db.add(asset)

app/dda/upload_io.py

@@ -0,0 +1,49 @@
+"""Stream large uploads to disk without loading into memory."""
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+
+from fastapi import HTTPException, UploadFile
+
+logger = logging.getLogger(__name__)
+
+CHUNK_SIZE = 8 * 1024 * 1024  # 8 MB
+
+
+async def stream_upload_to_file(
+    upload: UploadFile,
+    dest_path: Path,
+    max_bytes: int,
+) -> int:
+    """
+    Write upload body to dest_path in chunks. Returns total bytes written.
+    Removes partial file on failure or oversize.
+    """
+    dest_path.parent.mkdir(parents=True, exist_ok=True)
+    total = 0
+    try:
+        with dest_path.open("wb") as out:
+            while True:
+                chunk = await upload.read(CHUNK_SIZE)
+                if not chunk:
+                    break
+                total += len(chunk)
+                if total > max_bytes:
+                    raise HTTPException(
+                        status_code=400,
+                        detail=f"File too large (max {max_bytes // (1024 * 1024)} MB)",
+                    )
+                out.write(chunk)
+    except HTTPException:
+        dest_path.unlink(missing_ok=True)
+        raise
+    except Exception as exc:
+        dest_path.unlink(missing_ok=True)
+        logger.exception("Upload stream failed")
+        raise HTTPException(status_code=500, detail=f"Upload failed: {exc}") from exc
+
+    if total == 0:
+        dest_path.unlink(missing_ok=True)
+        raise HTTPException(status_code=400, detail="File is empty")
+    return total

static/css/dda.css

@@ -137,3 +137,19 @@
 .dda-slot-label { display: block; font-weight: 600; margin-bottom: 0.5rem; }
 
 .dda-upload-form .location-row { margin-bottom: 0.5rem; }
+
+.dda-upload-progress { margin: 0.75rem 0 1rem; }
+.dda-progress-bar {
+  height: 8px;
+  background: var(--bg);
+  border-radius: 4px;
+  overflow: hidden;
+  border: 1px solid var(--border);
+  margin-bottom: 0.35rem;
+}
+.dda-progress-fill {
+  height: 100%;
+  width: 0%;
+  background: linear-gradient(90deg, var(--grad-start), var(--grad-end, #10b981));
+  transition: width 0.15s ease;
+}

static/js/dda/app.js

@@ -63,7 +63,9 @@ async function initDda() {
     ddaConfig = await ddaApi('GET', '/api/dda/config');
     const hint = document.getElementById('lib-config-hint');
     if (hint) {
-      hint.textContent = `Max ${ddaConfig.maxUploadMb} MB · GeoTIFF: ${ddaConfig.geotiffEnabled ? 'yes' : 'limited'}`;
+      const gb = ddaConfig.maxUploadGb;
+      const label = gb >= 1 ? `${gb} GB GeoTIFF` : `${ddaConfig.maxGeotiffMb || ddaConfig.maxUploadMb} MB`;
+      hint.textContent = `Max ${label} · GeoTIFF engine: ${ddaConfig.geotiffEnabled ? 'yes' : 'limited'}`;
     }
     ddaHierarchy = await ddaApi('GET', '/api/dda/hierarchy');
     if (typeof renderHierarchy === 'function') renderHierarchy(ddaHierarchy);

static/js/dda/library.js

@@ -79,6 +79,32 @@ function populateUploadSelects(data) {
   }
 }
 
+function formatBytes(n) {
+  if (n >= 1024 ** 3) return (n / 1024 ** 3).toFixed(1) + ' GB';
+  if (n >= 1024 ** 2) return (n / 1024 ** 2).toFixed(1) + ' MB';
+  return (n / 1024).toFixed(0) + ' KB';
+}
+
+function uploadWithProgress(url, formData, onProgress) {
+  return new Promise((resolve, reject) => {
+    const xhr = new XMLHttpRequest();
+    xhr.open('POST', url);
+    xhr.withCredentials = true;
+    xhr.upload.addEventListener('progress', (e) => {
+      if (e.lengthComputable && onProgress) onProgress(e.loaded, e.total);
+    });
+    xhr.addEventListener('load', () => {
+      let data = null;
+      try { data = xhr.responseText ? JSON.parse(xhr.responseText) : null; } catch (_) {}
+      if (xhr.status >= 200 && xhr.status < 300) resolve(data);
+      else reject(new Error(data?.detail || xhr.statusText || 'Upload failed'));
+    });
+    xhr.addEventListener('error', () => reject(new Error('Network error during upload')));
+    xhr.addEventListener('abort', () => reject(new Error('Upload cancelled')));
+    xhr.send(formData);
+  });
+}
+
 document.getElementById('form-upload')?.addEventListener('submit', async (e) => {
   e.preventDefault();
   hideDdaError?.();
@@ -100,14 +126,29 @@ document.getElementById('form-upload')?.addEventListener('submit', async (e) =>
   form.append('manual_bounds_json', document.getElementById('up-manual-bounds').value || '');
 
   const btn = document.getElementById('btn-upload');
+  const progWrap = document.getElementById('upload-progress');
+  const progFill = document.getElementById('upload-progress-fill');
+  const progLabel = document.getElementById('upload-progress-label');
+
   btn.disabled = true;
   btn.textContent = 'Uploading…';
+  progWrap?.classList.remove('hidden');
+  if (progFill) progFill.style.width = '0%';
+  if (progLabel) progLabel.textContent = `Uploading ${file.name} (${formatBytes(file.size)})… 0%`;
+
   try {
-    const data = await ddaApi('POST', '/api/dda/images/upload', { body: form });
+    const data = await uploadWithProgress('/api/dda/images/upload', form, (loaded, total) => {
+      const pct = total ? Math.round((loaded / total) * 100) : 0;
+      if (progFill) progFill.style.width = pct + '%';
+      if (progLabel) progLabel.textContent = `Uploading… ${pct}% (${formatBytes(loaded)} / ${formatBytes(total)})`;
+    });
+    if (progFill) progFill.style.width = '100%';
     showDdaSuccess?.(data?.status === 'success' ? 'Image uploaded to library.' : 'Upload complete.');
     document.getElementById('form-upload').reset();
     document.getElementById('up-year').value = '2025';
     fileInput.value = '';
+    const dateInput = document.getElementById('up-date');
+    if (dateInput) dateInput.value = new Date().toISOString().slice(0, 10);
     window.ddaState.hierarchy = await ddaApi('GET', '/api/dda/hierarchy');
     renderHierarchy(window.ddaState.hierarchy);
     populateUploadSelects(window.ddaState.hierarchy);
@@ -117,6 +158,7 @@ document.getElementById('form-upload')?.addEventListener('submit', async (e) =>
   } finally {
     btn.disabled = false;
     btn.textContent = 'Upload to Library';
+    setTimeout(() => progWrap?.classList.add('hidden'), 1500);
   }
 });
 

templates/index_dda.html

@@ -5,7 +5,7 @@
   <meta name="viewport" content="width=device-width, initial-scale=1.0" />
   <title>DDA Change Detection</title>
   <link rel="stylesheet" href="/static/css/style.css?v=30" />
-  <link rel="stylesheet" href="/static/css/dda.css?v=1" />
+  <link rel="stylesheet" href="/static/css/dda.css?v=2" />
 </head>
 <body>
   <div class="app dda-app">
@@ -67,11 +67,15 @@
                   <label for="up-source">Source</label>
                   <select id="up-source"><option value="satellite">Satellite</option><option value="drone">Drone</option></select>
                 </div>
-                <div class="form-group">
-                  <label for="up-file">Image file</label>
-                  <input type="file" id="up-file" accept=".tif,.tiff,.png,.jpg,.jpeg" required />
-                </div>
+              <div class="form-group">
+                <label for="up-file">Image file (.tif up to 2 GB)</label>
+                <input type="file" id="up-file" accept=".tif,.tiff,.png,.jpg,.jpeg" required />
               </div>
+            </div>
+            <div id="upload-progress" class="dda-upload-progress hidden">
+              <div class="dda-progress-bar"><div id="upload-progress-fill" class="dda-progress-fill"></div></div>
+              <span id="upload-progress-label" class="dim">Uploading… 0%</span>
+            </div>
               <div class="form-group">
                 <label for="up-manual-bounds">Manual bounds (WGS84)</label>
                 <input type="text" id="up-manual-bounds" placeholder="west,south,east,north — required if GeoTIFF has no georef" />
@@ -119,7 +123,7 @@
     </section>
   </div>
 
-  <script src="/static/js/dda/app.js?v=1"></script>
-  <script src="/static/js/dda/library.js?v=1"></script>
+  <script src="/static/js/dda/app.js?v=2"></script>
+  <script src="/static/js/dda/library.js?v=2"></script>
 </body>
 </html>