File size: 1,944 Bytes
deb589d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 | """Download and verify unchanged base files for redistribution; never train or upload."""
import hashlib
import json
import shutil
from pathlib import Path
from huggingface_hub import HfApi, hf_hub_download
ROOT = Path(__file__).resolve().parents[1]
MODEL = "LiquidAI/LFM2.5-350M"
REVISION = "9e6c6ccf47cd318696e137d381a7ded8fe4df09f"
FILES = ["model.safetensors", "config.json", "generation_config.json", "tokenizer.json", "tokenizer_config.json", "chat_template.jinja", "LICENSE"]
def sha256(path):
with path.open("rb") as source:
return hashlib.file_digest(source, "sha256").hexdigest()
def main():
info = HfApi().model_info(MODEL, revision=REVISION, files_metadata=True)
metadata = {f.rfilename: f for f in info.siblings}
code_license = ROOT / "LICENSE-CODE"
if not code_license.exists():
assert (ROOT / "LICENSE").read_text().startswith("MIT License")
shutil.copy2(ROOT / "LICENSE", code_license)
manifest = {"source_repository": MODEL, "source_revision": REVISION,
"weights_modified": False, "training_performed": False, "files": {}}
for name in FILES:
source = Path(hf_hub_download(MODEL, name, revision=REVISION))
digest = sha256(source)
remote = metadata[name]
if remote.lfs:
assert digest == remote.lfs.sha256, name
else:
content = source.read_bytes()
blob = hashlib.sha1(f"blob {len(content)}\0".encode() + content).hexdigest()
assert blob == remote.blob_id, name
destination = ROOT / name
shutil.copy2(source, destination)
assert sha256(destination) == digest, name
manifest["files"][name] = {"sha256": digest, "size_bytes": destination.stat().st_size}
print(f"Verified unchanged: {name}", flush=True)
(ROOT / "BASE_MODEL_MANIFEST.json").write_text(json.dumps(manifest, indent=2) + "\n")
if __name__ == "__main__":
main()
|