Spaces:
Running
Running
File size: 6,125 Bytes
c55595d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 | """Fetch pinned published detector checkpoints and their original backbones."""
import argparse
import hashlib
import json
import shutil
import urllib.request
from pathlib import Path
from huggingface_hub import hf_hub_download, snapshot_download
ROOT = Path(__file__).resolve().parents[1]
PGC_REV = "86424d6502f143c84a9df91f8336d2da07d0e235"
FC_REV = "ee3ed528de85ed44b4fcb4657e1df44955ed463c"
DINO_REV = "47b73eefe95e8d44ec3623f8890bd894b6ea2d6c"
MDMF_REV = "22d03f49d877b8a01d730284b5f73df555f5656f"
MDMF_SHA256 = "b9ae92006dcc837066eec8c8a9a26133f64281d6f6713e9eb6e0ccdc213f76dc"
CODEBOOK_SHA256 = "0a7cd9555750c5f08b1db0a5a2dc0d866ef195fcd05210bffe53b2c82cdc87ab"
CLIP_URL = "https://openaipublic.azureedge.net/clip/models/b8cca3fd41ae0c99ba7e8951adf17d267cdb84cd88be6f7c2e0eca1737a03836/ViT-L-14.pt"
UPSTREAM_REVISIONS = {
"PGC": "0c9b10f3b89964b804ad6097e61709f74e827fbf",
"ForensicConcept": "1a57fbcd5212a113341b7377f307897509e13f0d",
"MDMF": MDMF_REV,
}
def digest(path):
h = hashlib.sha256()
with Path(path).open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
h.update(chunk)
return h.hexdigest()
def copy_verified(source, target, expected=None):
target.parent.mkdir(parents=True, exist_ok=True)
if not target.is_file() or (expected and digest(target) != expected and Path(source) != target):
shutil.copyfile(source, target)
if target.stat().st_size == 0:
raise ValueError(f"Empty weight file: {target}")
checksum = digest(target)
if expected and checksum != expected:
raise ValueError(f"Checksum mismatch for {target}: {checksum}")
return checksum
def hf_file(repo, revision, file, target):
source = hf_hub_download(repo_id=repo, revision=revision, filename=file)
return copy_verified(Path(source), target, digest(source))
def download(url, target, expected=None):
target.parent.mkdir(parents=True, exist_ok=True)
if not target.is_file() or (expected and digest(target) != expected):
temporary = target.with_suffix(target.suffix + ".part")
urllib.request.urlretrieve(url, temporary)
temporary.replace(target)
return copy_verified(target, target, expected)
def dino_backbone(method):
primary = ROOT / "detectors" / "PGC" / "checkpoint" / "backbones" / "dinov2-large"
if not (primary / "config.json").is_file() or not (primary / "model.safetensors").is_file():
source = Path(snapshot_download(repo_id="facebook/dinov2-large", revision=DINO_REV,
allow_patterns=["config.json", "preprocessor_config.json", "model.safetensors"]))
primary.mkdir(parents=True, exist_ok=True)
for file in source.iterdir():
if file.is_file():
copy_verified(file, primary / file.name, digest(file))
if not (primary / "model.safetensors").is_file():
raise FileNotFoundError(f"DINOv2 model weights missing under {primary}")
if method == "MDMF":
link = ROOT / "detectors" / "MDMF" / "checkpoint" / "backbones" / "dinov2-large"
link.parent.mkdir(parents=True, exist_ok=True)
if not link.exists():
link.symlink_to(primary, target_is_directory=True)
return {str(p.relative_to(ROOT)): digest(p) for p in primary.iterdir() if p.is_file()}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("method", choices=["PGC", "ForensicConcept", "MDMF", "all"])
args = parser.parse_args()
methods = ["PGC", "ForensicConcept", "MDMF"] if args.method == "all" else [args.method]
for method in methods:
base = ROOT / "detectors" / method / "checkpoint"
files = {}
if method in ("PGC", "MDMF"):
files.update(dino_backbone(method))
if method == "PGC":
path = base / "pretrained" / "weights" / "best.pt"
files[str(path.relative_to(ROOT))] = hf_file("xiaoyuzhou68/PGC_ckpt", PGC_REV,
"PGC_train_progan_sdv1_4_ckpt.pth", path)
source = f"https://huggingface.co/xiaoyuzhou68/PGC_ckpt/tree/{PGC_REV}"
license_name = "Apache-2.0"
elif method == "ForensicConcept":
path = base / "pretrained" / "weights" / "best.pt"
files[str(path.relative_to(ROOT))] = hf_file("ethan225/ForensicConcept", FC_REV,
"weights/detectors/clip_vitl14_codebook_stage2.pth", path)
clip = base / "backbones" / "ViT-L-14.pt"
files[str(clip.relative_to(ROOT))] = download(CLIP_URL, clip,
"b8cca3fd41ae0c99ba7e8951adf17d267cdb84cd88be6f7c2e0eca1737a03836")
codebook = ROOT / "detectors" / method / "upstream" / "codebooks" / "cleandift_codebook.npy"
codebook_url = (f"https://raw.githubusercontent.com/EthanAdamm/FORENSICCONCEPT/"
f"{UPSTREAM_REVISIONS[method]}/assets/codebooks/cleandift_codebook.npy")
files[str(codebook.relative_to(ROOT))] = download(codebook_url, codebook, CODEBOOK_SHA256)
source = f"https://huggingface.co/ethan225/ForensicConcept/tree/{FC_REV}"
license_name = "Apache-2.0 detector; MIT OpenAI CLIP backbone"
else:
path = base / "pretrained" / "weights" / "best.pt"
url = f"https://raw.githubusercontent.com/ZBox1005/MDMF/{MDMF_REV}/checkpoints/model.pth"
files[str(path.relative_to(ROOT))] = download(url, path, MDMF_SHA256)
source = f"https://github.com/ZBox1005/MDMF/tree/{MDMF_REV}"
license_name = "MIT"
record = {"method": method, "source": source, "upstream_code_revision": UPSTREAM_REVISIONS[method],
"license": license_name,
"backbone": f"facebook/dinov2-large@{DINO_REV}" if method != "ForensicConcept" else CLIP_URL,
"sha256": files}
(base / "weights-manifest.json").write_text(json.dumps(record, indent=2) + "\n")
print(f"Verified {method}: {len(files)} files; manifest: {base / 'weights-manifest.json'}")
if __name__ == "__main__":
main()
|