Spaces:
Running
Running
File size: 4,216 Bytes
6f5cc9b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 | """
One-time script to push eval.yaml files to benchmark dataset repos on HuggingFace.
This registers each EEG benchmark dataset with HF's decentralized eval system
by creating the dataset repo (if needed) and uploading an eval.yaml file.
Usage:
python -m scripts.push_eval_yaml
python -m scripts.push_eval_yaml --dry-run
python -m scripts.push_eval_yaml --benchmark bcic2a
"""
import argparse
import sys
import logging
import yaml
from app.config.base import HF_TOKEN
from app.config.benchmarks import EEG_BENCHMARKS, EVALUATION_FRAMEWORK
from app.config.hf_config import API as hf_api
from app.core.formatting import LogFormatter
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
def generate_eval_yaml(benchmark) -> str:
"""Generate eval.yaml content for a benchmark dataset."""
eval_config = {
"name": f"{benchmark.display_name} {benchmark.category}",
"description": benchmark.description,
"evaluation_framework": EVALUATION_FRAMEWORK,
"tasks": [
{
"id": benchmark.task_id,
"config": benchmark.config,
"split": benchmark.split,
}
],
}
return yaml.dump(eval_config, default_flow_style=False, sort_keys=False)
def push_eval_yaml(
benchmark_key: str = None,
dry_run: bool = False,
):
"""Push eval.yaml to benchmark dataset repos.
Args:
benchmark_key: If specified, only push for this benchmark.
dry_run: If True, print what would happen without pushing.
"""
benchmarks = EEG_BENCHMARKS
if benchmark_key:
if benchmark_key not in benchmarks:
logger.error(f"Unknown benchmark key: {benchmark_key}")
logger.info(f"Available keys: {list(benchmarks.keys())}")
sys.exit(1)
benchmarks = {benchmark_key: benchmarks[benchmark_key]}
logger.info(LogFormatter.section("PUSHING EVAL.YAML TO BENCHMARK DATASETS"))
for key, benchmark in benchmarks.items():
dataset_id = benchmark.dataset_id
logger.info(LogFormatter.subsection(f"Processing: {dataset_id}"))
eval_yaml_content = generate_eval_yaml(benchmark)
if dry_run:
logger.info(f"[DRY RUN] Would create repo: {dataset_id} (type=dataset)")
logger.info(f"[DRY RUN] Would upload eval.yaml:\n{eval_yaml_content}")
continue
# Create the dataset repo if it doesn't exist
try:
hf_api.create_repo(
repo_id=dataset_id,
repo_type="dataset",
exist_ok=True,
)
logger.info(LogFormatter.success(f"Repo ready: {dataset_id}"))
except Exception as e:
logger.error(LogFormatter.error(f"Failed to create repo {dataset_id}", e))
continue
# Upload eval.yaml
try:
hf_api.upload_file(
path_or_fileobj=eval_yaml_content.encode("utf-8"),
path_in_repo="eval.yaml",
repo_id=dataset_id,
repo_type="dataset",
commit_message="Add eval.yaml for EEG Finetune Arena benchmark registration",
)
logger.info(LogFormatter.success(f"Uploaded eval.yaml to {dataset_id}"))
except Exception as e:
logger.error(
LogFormatter.error(f"Failed to upload eval.yaml to {dataset_id}", e)
)
logger.info(LogFormatter.section("DONE"))
def main():
parser = argparse.ArgumentParser(
description="Push eval.yaml files to HF benchmark dataset repos."
)
parser.add_argument(
"--dry-run",
action="store_true",
help="Preview without pushing to HF.",
)
parser.add_argument(
"--benchmark",
type=str,
default=None,
help="Only push for a specific benchmark key (e.g. bcic2a).",
)
args = parser.parse_args()
if not HF_TOKEN and not args.dry_run:
logger.error("HF_TOKEN environment variable is required (set it or use --dry-run)")
sys.exit(1)
push_eval_yaml(benchmark_key=args.benchmark, dry_run=args.dry_run)
if __name__ == "__main__":
main()
|