DouDou commited on
Upload data3/quick_batch_start.sh with huggingface_hub
Browse files- data3/quick_batch_start.sh +50 -0
data3/quick_batch_start.sh
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/bin/bash
|
| 2 |
+
# 快速启动批处理(仅准备和提交,不等待完成)
|
| 3 |
+
|
| 4 |
+
set -e
|
| 5 |
+
|
| 6 |
+
echo "🚀 快速启动 Batch API 任务"
|
| 7 |
+
echo "========================================"
|
| 8 |
+
|
| 9 |
+
# 配置
|
| 10 |
+
MIN_SCORE=60
|
| 11 |
+
MODEL="gpt-4o-mini"
|
| 12 |
+
INPUT_FILE="function_dataset_v2.csv"
|
| 13 |
+
BATCH_REQUESTS_FILE="batch_requests_full.jsonl"
|
| 14 |
+
|
| 15 |
+
# 步骤1: 准备批量请求
|
| 16 |
+
echo "📋 准备批量请求..."
|
| 17 |
+
python3 generate_problems_batch.py prepare \
|
| 18 |
+
--input $INPUT_FILE \
|
| 19 |
+
--output $BATCH_REQUESTS_FILE \
|
| 20 |
+
--min-score $MIN_SCORE \
|
| 21 |
+
--model $MODEL
|
| 22 |
+
|
| 23 |
+
REQUEST_COUNT=$(wc -l < $BATCH_REQUESTS_FILE)
|
| 24 |
+
echo ""
|
| 25 |
+
echo "✅ 已准备 $REQUEST_COUNT 个请求"
|
| 26 |
+
|
| 27 |
+
# 步骤2: 提交
|
| 28 |
+
echo ""
|
| 29 |
+
echo "🚀 提交批处理任务..."
|
| 30 |
+
python3 generate_problems_batch.py submit \
|
| 31 |
+
--input $BATCH_REQUESTS_FILE \
|
| 32 |
+
--model $MODEL \
|
| 33 |
+
--description "Scientific computing problems - $REQUEST_COUNT samples" \
|
| 34 |
+
| tee batch_submit_output.txt
|
| 35 |
+
|
| 36 |
+
# 提取 Batch ID
|
| 37 |
+
BATCH_ID=$(grep -oP 'batch_[a-zA-Z0-9_]+' batch_submit_output.txt | head -1)
|
| 38 |
+
|
| 39 |
+
if [ -n "$BATCH_ID" ]; then
|
| 40 |
+
echo "$BATCH_ID" > batch_id.txt
|
| 41 |
+
echo ""
|
| 42 |
+
echo "✅ 批处理已提交!"
|
| 43 |
+
echo "📝 Batch ID: $BATCH_ID (已保存到 batch_id.txt)"
|
| 44 |
+
echo ""
|
| 45 |
+
echo "监控命令:"
|
| 46 |
+
echo " python3 generate_problems_batch.py status $BATCH_ID"
|
| 47 |
+
echo ""
|
| 48 |
+
echo "或运行监控脚本:"
|
| 49 |
+
echo " ./monitor_batch.sh"
|
| 50 |
+
fi
|