File size: 4,680 Bytes
f80360c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 | #!/usr/bin/env python3
"""
HuggingFace Model Uploader for Stack 2.9
Uploads fine-tuned model to HuggingFace Hub with proper model card and tags.
"""
import argparse
import os
import sys
from pathlib import Path
try:
from huggingface_hub import HfApi, create_repo, upload_folder
from huggingface_hub.utils import RepoNotFoundError
except ImportError:
print("Error: huggingface_hub not installed. Install with: pip install huggingface_hub")
sys.exit(1)
def load_model_card(template_path: str, placeholders: dict) -> str:
"""Load model card template and replace placeholders."""
with open(template_path, 'r') as f:
content = f.read()
for key, value in placeholders.items():
content = content.replace(f"{{{{{key}}}}}", str(value))
return content
def get_model_files(model_path: str) -> list:
"""Get list of model files to upload."""
path = Path(model_path)
if not path.exists():
print(f"Warning: Model path {model_path} does not exist. Creating empty upload.")
return []
files = []
for f in path.rglob("*"):
if f.is_file():
files.append(str(f.relative_to(path)))
return files
def main():
parser = argparse.ArgumentParser(description="Upload Stack 2.9 model to HuggingFace Hub")
parser.add_argument("--model", type=str, required=True,
help="Path to the merged model directory")
parser.add_argument("--repo-id", type=str, required=True,
help="HuggingFace repo ID (e.g., username/stack-2.9-7b)")
parser.add_argument("--token", type=str, default=None,
help="HuggingFace token (or set HF_TOKEN env var)")
parser.add_argument("--private", action="store_true",
help="Create private repository")
parser.add_argument("--dry-run", action="store_true",
help="Show what would be uploaded without uploading")
args = parser.parse_args()
# Get token from args or environment
token = args.token or os.environ.get("HF_TOKEN")
if not token:
print("Error: HF_TOKEN not set. Set HF_TOKEN environment variable or use --token")
sys.exit(1)
api = HfApi(token=token)
# Get model card content
script_dir = Path(__file__).parent
readme_path = script_dir / "README_HF.md"
if not readme_path.exists():
print(f"Warning: README_HF.md not found at {readme_path}")
readme_content = "# Stack 2.9 Model\n\nFine-tuned coding assistant model."
else:
# Placeholder values - update these with actual metrics
placeholders = {
"base_model": "Qwen2.5-Coder-7B",
"training_examples": "50,000",
"lora_rank": "16",
"lora_alpha": "32",
"humaneval_score": "TBD",
"mbpp_score": "TBD",
"max_context_length": "32K",
}
readme_content = load_model_card(str(readme_path), placeholders)
# Create repo if it doesn't exist
repo_id = args.repo_id
try:
create_repo(repo_id, token=token, private=args.private, repo_type="model", exist_ok=True)
print(f"Repository '{repo_id}' created or already exists.")
except Exception as e:
print(f"Error creating repository: {e}")
sys.exit(1)
# Write model card locally for upload
model_path = Path(args.model)
model_card_path = model_path / "README.md"
if model_path.exists():
with open(model_card_path, 'w') as f:
f.write(readme_content)
print(f"Created model card at {model_card_path}")
if args.dry_run:
print("\n=== DRY RUN - Files that would be uploaded ===")
files = get_model_files(args.model)
for f in files[:20]: # Show first 20
print(f" {f}")
if len(files) > 20:
print(f" ... and {len(files) - 20} more files")
print(f"\nTotal: {len(files)} files")
print(f"\nModel card:\n{readme_content[:500]}...")
return
# Upload the model
print(f"\nUploading model from {args.model} to {repo_id}...")
try:
# Upload entire folder
operation = api.upload_folder(
folder_path=args.model,
repo_id=repo_id,
repo_type="model",
commit_message="Upload Stack 2.9 fine-tuned model"
)
print(f"\n✅ Successfully uploaded to https://huggingface.co/{repo_id}")
print(f"Operation: {operation}")
except Exception as e:
print(f"Error uploading: {e}")
sys.exit(1)
if __name__ == "__main__":
main() |