Upload data3/monitor_batch.sh with huggingface_hub

data3/monitor_batch.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+# 监控批处理任务状态
+
+set -e
+
+BATCH_ID_FILE="batch_id.txt"
+
+# 获取 Batch ID
+if [ -n "$1" ]; then
+    BATCH_ID=$1
+elif [ -f "$BATCH_ID_FILE" ]; then
+    BATCH_ID=$(cat $BATCH_ID_FILE)
+else
+    echo "❌ 错误: 请提供 Batch ID"
+    echo "用法: $0 <batch_id>"
+    echo "或确保 batch_id.txt 文件存在"
+    exit 1
+fi
+
+echo "🔍 监控 Batch 任务: $BATCH_ID"
+echo "按 Ctrl+C 退出监控"
+echo "========================================"
+
+while true; do
+    TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S')
+    echo ""
+    echo "[$TIMESTAMP]"
+    
+    STATUS_OUTPUT=$(python3 generate_problems_batch.py status $BATCH_ID)
+    echo "$STATUS_OUTPUT"
+    
+    # 检查是否完成
+    if echo "$STATUS_OUTPUT" | grep -q "Status: completed"; then
+        echo ""
+        echo "✅ 批处理已完成！"
+        echo ""
+        echo "下载结果:"
+        echo "  python3 generate_problems_batch.py download $BATCH_ID --output batch_results_raw.jsonl"
+        echo ""
+        echo "处理结果:"
+        echo "  python3 generate_problems_batch.py process --input batch_results_raw.jsonl --output programming_problems_batch.jsonl"
+        break
+    elif echo "$STATUS_OUTPUT" | grep -q "Status: failed"; then
+        echo ""
+        echo "❌ 批处理失败！"
+        break
+    elif echo "$STATUS_OUTPUT" | grep -q "Status: expired"; then
+        echo ""
+        echo "❌ 批处理已过期！"
+        break
+    fi
+    
+    echo "⏳ 等待 5 分钟后再次检查..."
+    sleep 300
+done