feat: SAM3 concept segmentation replaces SAM2

reconstruct_objects(image, prompt) — SAM3 (facebook/sam3, transformers)
segments the object described by a short noun phrase and the best-scoring
instance goes to SAM 3D Objects. Default prompt 'object' keeps the
zero-config UX. Drops the sam2 runtime install and iopath.

Models load lazily inside @spaces.GPU with global caches — both SAM3D's
and SAM2's constructors crashed under ZeroGPU's startup CUDA emulation
(device-mixing tensor ops), so module-level cuda loading is not usable here.

Requires the space token to have accepted the gated facebook/sam3 license.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

app.py

@@ -1,8 +1,8 @@
 """
 SAM 3D Objects MCP Server
-Image → 3D Object (GLB)
+Image (+ text prompt) → 3D Object (GLB)
 
-SAM2 auto-detection + SAM 3D Objects reconstruction on ZeroGPU.
+SAM3 concept segmentation + SAM 3D Objects reconstruction on ZeroGPU.
 
 torch is provided by ZeroGPU. kaolin and pytorch3d are replaced by pure-python
 stubs covering exactly the surface the pipeline touches — texture baking, mesh
@@ -56,7 +56,6 @@ def _pip(*args):
 print("=== Runtime installs (need torch present) ===")
 # utils3d pinned to the commit MoGe expects — newer commits dropped points_to_normals
 _pip("--no-deps", "git+https://github.com/EasternJournalist/utils3d.git@3913c65d81e05e47b9f367250cf8c0f7462a0900")
-_pip("--no-deps", "sam2>=1.1.0")
 _pip("--no-deps", "git+https://github.com/microsoft/MoGe.git@a8c37341bc0325ca99b9d57981cc3bb2bd3e255b")
 # no prebuilt gsplat wheels beyond torch 2.4 — the PyPI sdist installs in JIT
 # mode; kernels never compile here because rendering paths are disabled
@@ -80,35 +79,53 @@ if hf_ckpt.exists() and not local_ckpt.exists():
     local_ckpt.symlink_to(hf_ckpt)
 CONFIG_PATH = str(local_ckpt / "pipeline.yaml")
 
-print("Loading models (ZeroGPU CUDA emulation)...")
-from sam2.automatic_mask_generator import SAM2AutomaticMaskGenerator
-from sam3d_inference import SAM3DInference
-
-# build on CPU first: SAM2's init runs tensor ops that mix devices under
-# CUDA emulation; a plain .to("cuda") placement afterwards is supported
-SAM2_GENERATOR = SAM2AutomaticMaskGenerator.from_pretrained(
-    "facebook/sam2-hiera-small", device="cpu")
-SAM2_GENERATOR.predictor.model.to("cuda")
-print("✓ SAM2 loaded")
-
-try:
-    SAM3D = SAM3DInference(CONFIG_PATH)
-    print("✓ SAM 3D Objects loaded (startup)")
-except Exception:
-    import traceback
-    traceback.print_exc()
-    SAM3D = None
-    print("! SAM3D startup load failed under CUDA emulation — lazy-loading in GPU context instead")
 print("=== Startup complete ===")
 
+# Model construction needs a real GPU (the constructors run device-mixing
+# tensor ops that ZeroGPU's startup CUDA emulation rejects), so models load
+# inside @spaces.GPU and are cached for reuse.
+SAM3 = None
+SAM3D = None
+
+
+def _get_sam3():
+    global SAM3
+    if SAM3 is None:
+        import torch
+        from transformers import Sam3Model, Sam3Processor
+        model = Sam3Model.from_pretrained("facebook/sam3").to("cuda").eval()
+        processor = Sam3Processor.from_pretrained("facebook/sam3")
+        SAM3 = (model, processor)
+    return SAM3
+
 
 def _get_sam3d():
     global SAM3D
     if SAM3D is None:
+        from sam3d_inference import SAM3DInference
         SAM3D = SAM3DInference(CONFIG_PATH)
     return SAM3D
 
 
+def _segment(image_np, prompt):
+    """SAM3 concept segmentation: returns the best-scoring mask for the
+    prompt as a bool array, plus the number of instances found."""
+    import torch
+    model, processor = _get_sam3()
+    inputs = processor(images=image_np, text=prompt, return_tensors="pt").to(model.device)
+    with torch.no_grad():
+        outputs = model(**inputs)
+    results = processor.post_process_instance_segmentation(
+        outputs, threshold=0.5, mask_threshold=0.5,
+        target_sizes=inputs.get("original_sizes").tolist(),
+    )[0]
+    masks = results["masks"]
+    if len(masks) == 0:
+        return None, 0
+    best = int(torch.as_tensor(results["scores"]).argmax())
+    return masks[best].cpu().numpy().astype(bool), len(masks)
+
+
 def _export_result(result, out_dir):
     """Export the pipeline result: prefer the ready-made GLB mesh, fall back
     to the raw gaussian splat PLY."""
@@ -136,43 +153,45 @@ def diagnose():
     lines = [f"torch={torch.__version__}", f"cuda={torch.cuda.is_available()}"]
     if torch.cuda.is_available():
         lines.append(f"gpu={torch.cuda.get_device_name()}")
-    lines.append(f"SAM2: {'loaded' if SAM2_GENERATOR is not None else 'missing'}")
-    lines.append(f"SAM3D: {'loaded' if SAM3D is not None else 'lazy (loads on first reconstruct)'}")
+    lines.append(f"SAM3: {'loaded' if SAM3 is not None else 'loads on first reconstruct'}")
+    lines.append(f"SAM3D: {'loaded' if SAM3D is not None else 'loads on first reconstruct'}")
     lines.append(f"config: {Path(CONFIG_PATH).exists()}")
     return "\n".join(lines)
 
 
 @spaces.GPU(duration=180)
-def reconstruct_objects(image: np.ndarray, progress=gr.Progress()):
+def reconstruct_objects(image: np.ndarray, prompt: str = "object", progress=gr.Progress()):
     """
-    Automatically detect the largest object with SAM2 and reconstruct it
-    in 3D with SAM 3D Objects.
+    Segment the object described by the text prompt with SAM3 and
+    reconstruct it in 3D with SAM 3D Objects.
 
     Args:
         image: Input RGB image
+        prompt: Short noun phrase describing what to reconstruct
+            (e.g. "the chair"). Default "object" picks the most
+            prominent object.
 
     Returns:
         tuple: (model_path, preview_image, status)
     """
     if image is None:
         return None, None, "❌ No image provided"
+    prompt = (prompt or "object").strip()
     try:
         import time
         import torch
         t0 = time.time()
         print(f"GPU: {torch.cuda.get_device_name()}")
 
-        progress(0.05, desc="Detecting objects (SAM2)...")
+        progress(0.05, desc=f"Segmenting '{prompt}' (SAM3)...")
         image_np = np.asarray(image)
-        masks = SAM2_GENERATOR.generate(image_np)
-        if not masks:
-            return None, image_np, "⚠️ No objects detected"
-        masks = sorted(masks, key=lambda x: x["area"], reverse=True)
-        best_mask = masks[0]["segmentation"]
+        best_mask, n_masks = _segment(image_np, prompt)
+        if best_mask is None:
+            return None, image_np, f"⚠️ No '{prompt}' found in image"
 
         preview = image_np.copy()
         preview[best_mask] = (preview[best_mask] * 0.5 + np.array([0, 255, 0]) * 0.5).astype(np.uint8)
-        print(f"  {len(masks)} masks ({time.time()-t0:.0f}s)")
+        print(f"  {n_masks} instances of '{prompt}' ({time.time()-t0:.0f}s)")
 
         progress(0.25, desc="Reconstructing 3D (SAM 3D Objects, ~1-2 min)...")
         result = _get_sam3d()(image=image_np, mask=best_mask, seed=42)
@@ -192,7 +211,7 @@ def reconstruct_objects(image: np.ndarray, progress=gr.Progress()):
             n_faces = len(trimesh.load(model_path, force="mesh").faces)
         except Exception:
             n_faces = 0
-        return model_path, preview, f"✓ {len(masks)} objects detected, {n_faces:,} faces ({int(time.time()-t0)}s)"
+        return model_path, preview, f"✓ {n_masks} × '{prompt}' found, {n_faces:,} faces ({int(time.time()-t0)}s)"
     except Exception:
         import traceback
         tb = traceback.format_exc()
@@ -203,23 +222,27 @@ def reconstruct_objects(image: np.ndarray, progress=gr.Progress()):
 with gr.Blocks(title="SAM 3D Objects MCP") as demo:
     gr.Markdown("""
     # 📦 SAM 3D Objects MCP Server
-    **Image → 3D Object (GLB)**
+    **Image (+ text prompt) → 3D Object (GLB)**
 
-    Automatically detects objects and reconstructs the largest one in 3D.
+    SAM3 segments the object you describe, SAM 3D Objects reconstructs it in 3D.
     """)
 
     with gr.Tab("Reconstruct"):
         with gr.Row():
             with gr.Column():
                 input_image = gr.Image(label="Input Image", type="numpy")
-                btn = gr.Button("🚀 Detect & Reconstruct", variant="primary", size="lg")
+                input_prompt = gr.Textbox(
+                    label="What to reconstruct", value="object",
+                    placeholder="e.g. the chair, yellow car, laptop")
+                btn = gr.Button("🚀 Segment & Reconstruct", variant="primary", size="lg")
             with gr.Column():
-                preview = gr.Image(label="Detected Object", type="numpy", interactive=False)
+                preview = gr.Image(label="Segmented Object", type="numpy", interactive=False)
                 status = gr.Textbox(label="Status")
         with gr.Row():
             output_model = gr.Model3D(label="3D Preview")
             output_file = gr.File(label="Download")
-        btn.click(reconstruct_objects, inputs=[input_image], outputs=[output_model, preview, status])
+        btn.click(reconstruct_objects, inputs=[input_image, input_prompt],
+                  outputs=[output_model, preview, status])
         output_model.change(lambda x: x, inputs=[output_model], outputs=[output_file])
 
     with gr.Tab("Diagnose"):

requirements.txt

@@ -24,6 +24,9 @@ huggingface_hub
 timm
 astor
 
+# SAM3 segmentation
+transformers
+
 # 3D geometry / mesh processing
 trimesh
 plyfile
@@ -35,6 +38,3 @@ igraph
 
 # sparse convolution — standalone binary (cumm runtime), independent of torch version
 spconv-cu124==2.3.8
-
-# sam2 runtime dependency
-iopath