Fix local library scan: DDA mode in run.py, path encoding, gitignore large tifs

.gitignore

@@ -4,6 +4,12 @@ __pycache__/
 data/satellite_app.db
 data/overlays/*
 !data/overlays/.gitkeep
+# Local library images (too large for git — keep on disk only)
+library_sources/**/*.tif
+library_sources/**/*.tiff
+library_sources/**/*.TIF
+library_sources/**/*.TIFF
+!library_sources/**/.gitkeep
 .env
 *.egg-info/
 dist/

Dockerfile

@@ -21,7 +21,7 @@ WORKDIR /app
 
 # Build-time info + cache-bust:
 # Changing APP_BUILD forces Docker to re-run subsequent layers (including pip install).
-ARG APP_BUILD=27
+ARG APP_BUILD=28
 ENV APP_BUILD=${APP_BUILD}
 ENV GDAL_CONFIG=/usr/bin/gdal-config
 RUN echo "Docker build start: APP_BUILD=${APP_BUILD}" && python -V

app/dda/config.py

@@ -1,5 +1,6 @@
 import os
 from pathlib import Path
+from typing import List
 
 from ..database import DATA_DIR
 
@@ -9,10 +10,21 @@ IS_DDA_MODE = APP_MODE == "dda"
 # Project root: change_detection_webapp/
 PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
 
-# Local folder library — drop images into year subfolders here (no web upload required)
-LOCAL_LIBRARY_ROOT = Path(
-    os.environ.get("LOCAL_LIBRARY_ROOT", str(PROJECT_ROOT / "library_sources"))
-).resolve()
+
+def get_library_roots() -> List[Path]:
+    """All folders scanned for year-based images (project + writable data copy)."""
+    roots: List[Path] = []
+    if os.environ.get("LOCAL_LIBRARY_ROOT"):
+        roots.append(Path(os.environ["LOCAL_LIBRARY_ROOT"]).resolve())
+    for candidate in (PROJECT_ROOT / "library_sources", DATA_DIR / "library_sources"):
+        resolved = candidate.resolve()
+        if resolved not in roots:
+            roots.append(resolved)
+    return roots
+
+
+# Primary root (shown in UI) — first entry from get_library_roots()
+LOCAL_LIBRARY_ROOT = get_library_roots()[0]
 
 LIBRARY_DIR = DATA_DIR / "library"
 THUMBS_DIR = LIBRARY_DIR / "thumbs"
@@ -29,7 +41,12 @@ ALLOWED_EXTENSIONS = {".tif", ".tiff", ".png", ".jpg", ".jpeg"}
 
 
 def ensure_library_dirs() -> None:
-    for d in (LIBRARY_DIR, THUMBS_DIR, PREVIEWS_DIR, LOCAL_THUMB_CACHE, LOCAL_LIBRARY_ROOT):
+    for root in get_library_roots():
+        try:
+            root.mkdir(parents=True, exist_ok=True)
+        except OSError:
+            pass
+    for d in (LIBRARY_DIR, THUMBS_DIR, PREVIEWS_DIR, LOCAL_THUMB_CACHE):
         try:
             d.mkdir(parents=True, exist_ok=True)
         except OSError:
@@ -37,14 +54,15 @@ def ensure_library_dirs() -> None:
 
 
 def ensure_local_year_folders() -> None:
-    """Create default year folders under library_sources if missing."""
+    """Create default year folders under each library root if missing."""
     from datetime import datetime
     current = datetime.now().year
-    for year in (current - 1, current, current + 1):
-        try:
-            (LOCAL_LIBRARY_ROOT / str(year)).mkdir(parents=True, exist_ok=True)
-        except OSError:
-            pass
+    for root in get_library_roots():
+        for year in (current - 1, current, current + 1):
+            try:
+                (root / str(year)).mkdir(parents=True, exist_ok=True)
+            except OSError:
+                pass
 
 
 def max_upload_bytes_for_extension(ext: str) -> int:

app/dda/library_routes.py

@@ -65,7 +65,7 @@ def _image_to_dict(asset: ImageAsset, zone_name: str = "", village_name: str = "
 @router.get("/config")
 def dda_config():
     _require_dda()
-    from .config import MAX_GEOTIFF_BYTES, MAX_IMAGE_BYTES, LOCAL_LIBRARY_ROOT
+    from .config import MAX_GEOTIFF_BYTES, MAX_IMAGE_BYTES, get_library_roots
     max_gb = MAX_GEOTIFF_BYTES / (1024 ** 3)
     return {
         "mode": "dda",
@@ -77,7 +77,7 @@ def dda_config():
         "allowedExtensions": sorted(ALLOWED_EXTENSIONS),
         "hierarchyMode": "admin",
         "librarySource": "local_folder",
-        "localLibraryPath": str(LOCAL_LIBRARY_ROOT),
+        "localLibraryPaths": [str(r) for r in get_library_roots()],
     }
 
 

app/dda/local_library.py

@@ -6,12 +6,10 @@ Folder layout (under library_sources/):
     library_sources/
       2024/
         image_a.tif
-        site_1/
-          image_b.tif
       2025/
         image_c.tif
 
-Copy or save files directly into the year folder on disk; the app scans on load / refresh.
+Copy files into library_sources/YEAR/ in the project folder (or data/library_sources/ on HF).
 """
 from __future__ import annotations
 
@@ -20,10 +18,11 @@ import logging
 from dataclasses import dataclass
 from pathlib import Path
 from typing import List, Optional
+from urllib.parse import quote
 
 from fastapi import HTTPException
 
-from .config import ALLOWED_EXTENSIONS, LOCAL_LIBRARY_ROOT, LOCAL_THUMB_CACHE
+from .config import ALLOWED_EXTENSIONS, LOCAL_THUMB_CACHE, get_library_roots
 from .geotiff_io import inspect_image, raster_to_preview_png
 
 logger = logging.getLogger(__name__)
@@ -32,6 +31,7 @@ logger = logging.getLogger(__name__)
 @dataclass
 class LocalImageEntry:
     path: str  # relative posix path, e.g. 2025/aerial.tif
+    root: Path
     year: int
     filename: str
     file_size_bytes: int
@@ -41,73 +41,80 @@ def _is_year_dir(name: str) -> bool:
     return len(name) == 4 and name.isdigit() and 1990 <= int(name) <= 2100
 
 
+def _iter_image_files(root: Path, year: Optional[int] = None):
+    if not root.exists():
+        return
+    if year is not None:
+        year_dirs = [root / str(year)] if (root / str(year)).is_dir() else []
+    else:
+        year_dirs = [d for d in sorted(root.iterdir()) if d.is_dir() and _is_year_dir(d.name)]
+    for ydir in year_dirs:
+        y = int(ydir.name)
+        for path in sorted(ydir.rglob("*")):
+            if not path.is_file():
+                continue
+            ext = path.suffix.lower()
+            if ext not in ALLOWED_EXTENSIONS:
+                logger.debug("Skipped (extension %s): %s", ext, path)
+                continue
+            yield root, y, path
+
+
 def safe_resolve(relative_path: str) -> Path:
-    """Resolve a library-relative path; block path traversal."""
+    """Resolve a library-relative path across all configured roots."""
     rel = relative_path.replace("\\", "/").lstrip("/")
     if not rel or ".." in rel.split("/"):
         raise HTTPException(status_code=400, detail="Invalid image path")
-    root = LOCAL_LIBRARY_ROOT.resolve()
-    full = (root / rel).resolve()
-    try:
-        full.relative_to(root)
-    except ValueError:
-        raise HTTPException(status_code=400, detail="Invalid image path")
-    if not full.is_file():
-        raise HTTPException(status_code=404, detail="Image file not found")
-    if full.suffix.lower() not in ALLOWED_EXTENSIONS:
-        raise HTTPException(status_code=400, detail="Unsupported image format")
-    return full
+    for root in get_library_roots():
+        full = (root / rel).resolve()
+        try:
+            full.relative_to(root.resolve())
+        except ValueError:
+            continue
+        if full.is_file() and full.suffix.lower() in ALLOWED_EXTENSIONS:
+            return full
+    raise HTTPException(status_code=404, detail="Image file not found")
 
 
 def scan_years() -> List[dict]:
-    """List year folders and image counts."""
+    """List year folders and image counts (merged across all roots)."""
     ensure_root()
-    years = []
-    if not LOCAL_LIBRARY_ROOT.exists():
-        return years
-    for entry in sorted(LOCAL_LIBRARY_ROOT.iterdir()):
-        if not entry.is_dir() or not _is_year_dir(entry.name):
+    counts: dict[int, int] = {}
+    for root in get_library_roots():
+        if not root.exists():
             continue
-        count = sum(
-            1
-            for p in entry.rglob("*")
-            if p.is_file() and p.suffix.lower() in ALLOWED_EXTENSIONS
-        )
-        years.append({"year": int(entry.name), "imageCount": count})
-    return years
+        for entry in sorted(root.iterdir()):
+            if not entry.is_dir() or not _is_year_dir(entry.name):
+                continue
+            y = int(entry.name)
+            n = sum(1 for _ in _iter_image_files(root, year=y))
+            counts[y] = counts.get(y, 0) + n
+    return [{"year": y, "imageCount": counts[y]} for y in sorted(counts)]
 
 
 def scan_images(year: Optional[int] = None, query: Optional[str] = None) -> List[LocalImageEntry]:
-    """Scan library_sources for images, optionally filtered by year and filename."""
+    """Scan all library roots for images."""
     ensure_root()
     results: List[LocalImageEntry] = []
-    root = LOCAL_LIBRARY_ROOT
-    if not root.exists():
-        return results
-
+    seen_paths: set[str] = set()
     q = (query or "").strip().lower()
-    year_dirs: List[Path]
-    if year is not None:
-        ydir = root / str(year)
-        year_dirs = [ydir] if ydir.is_dir() else []
-    else:
-        year_dirs = [d for d in sorted(root.iterdir()) if d.is_dir() and _is_year_dir(d.name)]
 
-    for ydir in year_dirs:
-        y = int(ydir.name)
-        for path in sorted(ydir.rglob("*")):
-            if not path.is_file() or path.suffix.lower() not in ALLOWED_EXTENSIONS:
+    for root in get_library_roots():
+        for r, y, path in _iter_image_files(root, year=year):
+            rel = path.relative_to(r).as_posix()
+            if rel in seen_paths:
                 continue
-            rel = path.relative_to(root).as_posix()
             if q and q not in rel.lower():
                 continue
             try:
                 size = path.stat().st_size
             except OSError:
                 continue
+            seen_paths.add(rel)
             results.append(
                 LocalImageEntry(
                     path=rel,
+                    root=r,
                     year=y,
                     filename=path.name,
                     file_size_bytes=size,
@@ -117,13 +124,15 @@ def scan_images(year: Optional[int] = None, query: Optional[str] = None) -> List
 
 
 def entry_to_dict(entry: LocalImageEntry, include_meta: bool = False) -> dict:
+    encoded_path = quote(entry.path, safe="/")
     out = {
         "path": entry.path,
         "year": entry.year,
         "filename": entry.filename,
         "fileSizeBytes": entry.file_size_bytes,
-        "thumbUrl": f"/api/dda/local/thumb?path={entry.path}",
+        "thumbUrl": f"/api/dda/local/thumb?path={encoded_path}",
         "source": "local_folder",
+        "rootPath": str(entry.root),
     }
     if include_meta:
         try:
@@ -148,7 +157,7 @@ def thumb_cache_path(relative_path: str) -> Path:
 def get_or_build_thumb(relative_path: str, max_side: int = 256) -> Path:
     full = safe_resolve(relative_path)
     cache = thumb_cache_path(relative_path)
-    if cache.exists():
+    if cache.exists() and cache.stat().st_mtime >= full.stat().st_mtime:
         return cache
     cache.parent.mkdir(parents=True, exist_ok=True)
     raster_to_preview_png(full, cache, max_side=max_side)
@@ -156,5 +165,33 @@ def get_or_build_thumb(relative_path: str, max_side: int = 256) -> Path:
 
 
 def ensure_root() -> None:
-    LOCAL_LIBRARY_ROOT.mkdir(parents=True, exist_ok=True)
+    for root in get_library_roots():
+        root.mkdir(parents=True, exist_ok=True)
     LOCAL_THUMB_CACHE.mkdir(parents=True, exist_ok=True)
+
+
+def library_debug_info() -> dict:
+    """Diagnostics for troubleshooting missing images."""
+    roots_info = []
+    for root in get_library_roots():
+        info = {
+            "path": str(root),
+            "exists": root.exists(),
+            "years": [],
+            "otherFiles": [],
+        }
+        if root.exists():
+            for entry in sorted(root.iterdir()):
+                if entry.is_dir() and _is_year_dir(entry.name):
+                    files = []
+                    for _, _, p in _iter_image_files(root, year=int(entry.name)):
+                        files.append({"name": p.name, "size": p.stat().st_size})
+                    info["years"].append({"year": entry.name, "files": files})
+                elif entry.is_file() and entry.name not in ("README.md", ".gitkeep"):
+                    info["otherFiles"].append(entry.name)
+        roots_info.append(info)
+    return {
+        "roots": roots_info,
+        "allowedExtensions": sorted(ALLOWED_EXTENSIONS),
+        "totalImages": len(scan_images()),
+    }

app/dda/local_routes.py

@@ -4,10 +4,11 @@ from typing import Optional
 from fastapi import APIRouter, HTTPException, Query
 from fastapi.responses import FileResponse
 
-from .config import IS_DDA_MODE, LOCAL_LIBRARY_ROOT, geotiff_io_available
+from .config import IS_DDA_MODE, geotiff_io_available, get_library_roots
 from .local_library import (
     entry_to_dict,
     get_or_build_thumb,
+    library_debug_info,
     safe_resolve,
     scan_images,
     scan_years,
@@ -24,21 +25,30 @@ def _require_dda():
 @router.get("/local/config")
 def local_library_config():
     _require_dda()
+    roots = [str(r) for r in get_library_roots()]
     return {
         "source": "local_folder",
-        "rootPath": str(LOCAL_LIBRARY_ROOT),
+        "rootPath": roots[0] if roots else "",
+        "rootPaths": roots,
         "instructions": (
-            "Copy .tif / .tiff images into year folders under library_sources/ "
-            "(e.g. library_sources/2025/my_image.tif), then click Refresh."
+            "Copy .tif / .tiff images into library_sources/YEAR/ inside the project folder "
+            "(e.g. change_detection_webapp/library_sources/2025/), then click Refresh. "
+            "Run locally with: python run.py (DDA mode is enabled automatically)."
         ),
         "geotiffEnabled": geotiff_io_available(),
     }
 
 
+@router.get("/local/debug")
+def local_debug():
+    _require_dda()
+    return library_debug_info()
+
+
 @router.get("/local/years")
 def local_years():
     _require_dda()
-    return {"years": scan_years(), "rootPath": str(LOCAL_LIBRARY_ROOT)}
+    return {"years": scan_years(), "rootPaths": [str(r) for r in get_library_roots()]}
 
 
 @router.get("/local/images")
@@ -82,4 +92,4 @@ def local_rescan():
     _require_dda()
     years = scan_years()
     total = sum(y["imageCount"] for y in years)
-    return {"ok": True, "years": years, "totalImages": total, "rootPath": str(LOCAL_LIBRARY_ROOT)}
+    return {"ok": True, "years": years, "totalImages": total, "rootPaths": [str(r) for r in get_library_roots()]}

library_sources/README.md

@@ -22,10 +22,25 @@ library_sources/
 
 ## How to use
 
-1. Copy your images into the correct **year** folder (e.g. `library_sources/2025/`).
-2. Run the app (`python run.py` or open the dev Space).
-3. Open **Image Library** → click **Refresh** if you added files while the app was running.
-4. Select a year in the sidebar to view images.
+1. Copy your images into the correct **year** folder:
+   ```
+   change_detection_webapp/library_sources/2025/your_image.tif
+   change_detection_webapp/library_sources/2026/other_image.tif
+   ```
+2. Start the app locally: `python run.py` (DDA mode is enabled automatically).
+3. Open **Image Library** → click **Refresh**.
+4. Select **2025** or **2026** in the sidebar to filter by year.
+
+> **Hugging Face dev Space:** large `.tif` files are **not** uploaded via git. Run locally, or copy files into the Space persistent `data/library_sources/` folder.
+
+## Troubleshooting
+
+| Problem | Fix |
+|---------|-----|
+| Images not showing | Use `python run.py` locally; click **Refresh** |
+| Wrong page (simple upload UI) | Set `APP_MODE=dda` on HF, or run locally |
+| Filename has spaces | Supported (e.g. `Grid no. 54_ORI-005.tif`) |
+| Check scan | Open `/api/dda/local/debug` in browser |
 
 ## Large files
 

run.py

@@ -35,6 +35,9 @@ def main():
     if here not in sys.path:
         sys.path.insert(0, here)
 
+    # Local runs use DDA dev UI + folder library unless already set
+    os.environ.setdefault("APP_MODE", "dda")
+
     try:
         import uvicorn
     except ImportError:

static/js/dda/app.js

@@ -77,10 +77,11 @@ async function initDda() {
     if (hint) {
       hint.textContent = localCfg.geotiffEnabled ? 'GeoTIFF ready' : 'GeoTIFF limited';
     }
+    const paths = (localCfg.rootPaths || [localCfg.rootPath || ddaConfig.localLibraryPaths?.[0]]).filter(Boolean);
     const pathEl = document.getElementById('lib-path-display');
-    if (pathEl) pathEl.textContent = localCfg.rootPath || ddaConfig.localLibraryPath || '';
+    if (pathEl) pathEl.textContent = paths.join('\n');
     const folderPath = document.getElementById('lib-folder-path');
-    if (folderPath) folderPath.textContent = 'Folder: library_sources/';
+    if (folderPath) folderPath.textContent = paths.length ? `Scanning: ${paths[0]}` : '';
 
     const yearsData = await ddaApi('GET', '/api/dda/local/years');
     localYears = yearsData.years || [];
@@ -108,15 +109,18 @@ async function loadLibraryImages() {
       grid.innerHTML = `<p class="dim">No images in ${selectedYear || 'library_sources'}. Copy .tif files into <code>library_sources/${selectedYear || 'YEAR'}/</code> and click Refresh.</p>`;
       return;
     }
-    grid.innerHTML = items.map((img) => `
-      <div class="dda-card-img" draggable="true" data-image-path="${img.path}" title="${img.filename}">
-        ${img.thumbUrl ? `<img src="${img.thumbUrl}" alt="" loading="lazy" />` : '<div class="meta">No preview</div>'}
+    grid.innerHTML = items.map((img) => {
+      const thumb = img.thumbUrl ? img.thumbUrl.replace(/path=[^&]+/, 'path=' + encodeURIComponent(img.path)) : '';
+      return `
+      <div class="dda-card-img" draggable="true" data-image-path="${img.path.replace(/"/g, '&quot;')}" title="${img.filename}">
+        ${thumb ? `<img src="${thumb}" alt="" loading="lazy" />` : '<div class="meta">No preview</div>'}
         <div class="meta">
           <strong>${img.year}</strong><br/>
           ${img.filename}<br/>
           <span class="dim">${formatBytes(img.fileSizeBytes)}</span>
         </div>
-      </div>`).join('');
+      </div>`;
+    }).join('');
     grid.querySelectorAll('.dda-card-img').forEach((card) => {
       card.addEventListener('dragstart', (e) => {
         e.dataTransfer.setData('application/x-dda-image-path', card.dataset.imagePath);

templates/index_dda.html

@@ -88,7 +88,7 @@
     </section>
   </div>
 
-  <script src="/static/js/dda/app.js?v=3"></script>
-  <script src="/static/js/dda/library.js?v=3"></script>
+  <script src="/static/js/dda/app.js?v=4"></script>
+  <script src="/static/js/dda/library.js?v=4"></script>
 </body>
 </html>