remove torch.hub DINOv2 download: stub out encoder (output was always zeros), strict=False to skip DINO checkpoint keys

ldm/modules/encoders/modules.py

@@ -263,12 +263,13 @@ class DINOEmbedder(AbstractEncoder):
         #     'huge': 1536
         # }
         
-        # embedding_size = embedding_sizes[dino_version]
-        letter = letter_map[dino_version]
-        # self.transformer = CLIPVisionModel.from_pretrained(version)
-        self.dino_model = torch.hub.load('facebookresearch/dinov2', f'dinov2_vit{letter}14_reg', pretrained=False).cuda()
-
-
+        embedding_sizes = {
+            'small': 384,
+            'big': 768,
+            'large': 1024,
+            'huge': 1536
+        }
+        self.embedding_dim = embedding_sizes[dino_version]
         self.freeze()
 
     def freeze(self):
@@ -276,12 +277,12 @@ class DINOEmbedder(AbstractEncoder):
             param.requires_grad = False
 
     def forward(self, image):
-        with torch.no_grad():
-            outputs = self.dino_model.forward_features(image)
-        patch_tokens = outputs['x_norm_patchtokens']
-        global_token = outputs['x_norm_clstoken'].unsqueeze(1)
-        features = torch.concat([patch_tokens, global_token], dim=1)
-        return torch.zeros_like(features)
+        # DINO output is unused (returns zeros); compute shape from input without loading the model
+        B = image.shape[0]
+        patch_size = 14
+        h, w = image.shape[-2], image.shape[-1]
+        num_patches = (h // patch_size) * (w // patch_size)
+        return torch.zeros(B, num_patches + 1, self.embedding_dim, device=image.device, dtype=image.dtype)
 
     def encode(self, image):
         return self(image)

run_magicfu.py

@@ -115,7 +115,7 @@ def get_model(config_path, ckpt_path):
     model = load_model_from_config(config, None)
     pl_sd = torch.load(ckpt_path, map_location="cpu")
     
-    m, u = model.load_state_dict(pl_sd, strict=True)
+    m, u = model.load_state_dict(pl_sd, strict=False)
     if len(m) > 0:
         print("WARNING: missing keys:")
         print(m)