dev/r5_bootstrap.py

#!/usr/bin/env python3
"""r5_bootstrap.py — 在新 T4 Colab 上一键恢复第五轮 sweep (自包含, 幂等)

用法 (通过 colabcli /execute_stream 运行本文件全部内容):
  1. 环境对齐: torchao 卸载, transformers==5.16.1 / peft==0.20.0 校准
  2. 从 HF 直连拉取: 模型权重(5GB) / round2 LoRA(out_zh) / round4 head / round3 数据标签
  3. 从 HF 代码仓 tchbcb/samaidev-pnet-2b 拉取 pondernet 包 + 平铺兼容垫片
  4. 内嵌启动 r5_sweep (A1..C + R4 复评, 幂等: 已有 head+dump 的阶段自动跳过)
"""
import base64
import json
import shutil
import os
import subprocess
import sys
import time

ROOT = "/content/pondernet_r5"
MODEL_DIR = f"{ROOT}/models/MiniCPM5-2B"
HF_MODEL = "tchbcb/MiniCPM5-2B-cpu"
HF_CODE = "tchbcb/samaidev-pnet-2b"
os.makedirs(f"{ROOT}/data_zh", exist_ok=True)
os.chdir(ROOT)

SWEEP_B64 = "IyAtKi0gY29kaW5nOiB1dGYtOCAtKi0KIiIicjVfc3dlZXAucHkg4oCUIOesrOS6lOi9ruaAu+aOpzog5pa55ZCR4pGg5raI6J6NICsg4pGh5oCd6ICD5Z2X6YCC6YWNICsg4pGi562U5qGI57qnIFJMLCDorq3nu4PihpTor4TkvLDkuqTplJkKCumYtuautSAo5q+P6Zi25q615bmC562JOiBoZWFkK2R1bXAg5Z2H5a2Y5Zyo5YiZ6Lez6L+HKToKICBBMSDliIbmobbliqDmnYMgZWFzeTo4ICAoaGVhZC1vbmx5LCByb3VuZDQg55uu5qCHKQogIEEyIOWFqOWxgOadg+mHjSAxMCAgICAgIChoZWFkLW9ubHksIHJvdW5kNCDnm67moIcpCiAgQTMgc2hhcnAg55uu5qCHICsgbWFwIChoZWFkLW9ubHkpCiAgQTQgTUxQLTI1NiDlpLQgICAgICAoaGVhZC1vbmx5LCBzaGFycCArIG1hcCkKICBCICDpgJDmraVDRStMb1JBICAgICAgKOaWueWQkeKRoSwgaGVhZCtsb3JhIOe7reiurSBvdXRfemgsIGxyIDVlLTUsIHN0ZXAtY2UgMS4wKQogIEMgIOetlOahiOe6pyBSTCAgICAgICAgKOaWueWQkeKRoiwgUkVJTkZPUkNFLCA5NiDmoLfmnKwsIM67PTAuMykKICBSNCDlpI3or4Tln7rnur8gICAgICAgICAob3V0X3I0IGhlYWQg5ZCM5pel5aSN5rWLLCDmoKHlh4blr7nmr5Tlj6PlvoQpCiIiIgppbXBvcnQganNvbgppbXBvcnQgb3MKaW1wb3J0IHN1YnByb2Nlc3MKaW1wb3J0IHN5cwppbXBvcnQgdGltZQoKb3MuY2hkaXIoIi9jb250ZW50L3BvbmRlcm5ldF9yNCIpClBZID0gc3lzLmV4ZWN1dGFibGUKTU9ERUwgPSAibW9kZWxzL01pbmlDUE01LTJCIgpUUkFJTiA9ICJkYXRhX3poL2Jvb3RfdHJhaW4uanNvbmwiCkVWQUwgPSAiZGF0YV96aC9ib290X2V2YWwuanNvbmwiCgpIRUFEX0NPTU1PTiA9ICgiLS10cmFpbi1tb2RlIGhlYWQgLS1pbml0LWFkYXB0ZXIgb3V0X3poIC0tc3RlcC1zdXBlcnZpc2UgIgogICAgICAgICAgICAgICAiLS1kaWZmaWN1bHR5LXByaW9yIGVhc3k6MC45LG1lZGl1bTowLjUsaGFyZDowLjA1ICIKICAgICAgICAgICAgICAgIi0tZXBvY2hzIDIgLS1iYXRjaC1zaXplIDIgLS1hY2N1bSA4IC0tbWF4LXN0ZXBzIDYgIgogICAgICAgICAgICAgICAiLS1tYXgtbGVuIDc2OCAtLWdyYWQtY2twdCAtLWR0eXBlIGZsb2F0MTYgLS1sb2ctZXZlcnkgMTAiKQpUUkFJTl9DT01NT04gPSBmIntQWX0gdHJhaW5fcG9uZGVyX2hlYWQucHkgLS1tb2RlbCB7TU9ERUx9IC0tZGF0YSB7VFJBSU59IgoKU1RBR0VTID0gWwogICAgKCJBMV9tYXA4IiwgZiJ7VFJBSU5fQ09NTU9OfSB7SEVBRF9DT01NT059IC0tc3VwLXdlaWdodC1tYXAgZWFzeTo4LG1lZGl1bToyLGhhcmQ6MSAtLW91dHB1dCBvdXRfcjVfQTEiKSwKICAgICgiQTJfdzEwIiwgZiJ7VFJBSU5fQ09NTU9OfSB7SEVBRF9DT01NT059IC0tc3VwLXdlaWdodCAxMC4wIC0tb3V0cHV0IG91dF9yNV9BMiIpLAogICAgKCJBM19zaGFycCIsIGYie1RSQUlOX0NPTU1PTn0ge0hFQURfQ09NTU9OfSAtLXN1cC10YXJnZXQtcHJlc2V0IHNoYXJwIC0tc3VwLXdlaWdodC1tYXAgZWFzeTo0LG1lZGl1bToyLGhhcmQ6MSAtLW91dHB1dCBvdXRfcjVfQTMiKSwKICAgICgiQTRfbWxwIiwgZiJ7VFJBSU5fQ09NTU9OfSB7SEVBRF9DT01NT059IC0tc3VwLXRhcmdldC1wcmVzZXQgc2hhcnAgLS1zdXAtd2VpZ2h0LW1hcCBlYXN5OjQsbWVkaXVtOjIsaGFyZDoxIC0taGVhZC1tbHAgMjU2IC0tb3V0cHV0IG91dF9yNV9BNCIpLAogICAgKCJCX3N0ZXBjZSIsIGYie1RSQUlOX0NPTU1PTn0gLS10cmFpbi1tb2RlIGhlYWQrbG9yYSAtLWluaXQtYWRhcHRlciBvdXRfemggLS1zdGVwLXN1cGVydmlzZSAiCiAgICAgICAgICAgICAgICAgZiItLXN0ZXAtY2Utd2VpZ2h0IDEuMCAtLXN1cC13ZWlnaHQtbWFwIGVhc3k6NCxtZWRpdW06MixoYXJkOjEgLS1sci1sb3JhIDVlLTUgIgogICAgICAgICAgICAgICAgIGYiLS1lcG9jaHMgMSAtLWJhdGNoLXNpemUgMiAtLWFjY3VtIDggLS1tYXgtc3RlcHMgNiAtLW1heC1sZW4gNzY4ICIKICAgICAgICAgICAgICAgICBmIi0tZ3JhZC1ja3B0IC0tZHR5cGUgZmxvYXQxNiAtLWxvZy1ldmVyeSA1IC0tb3V0cHV0IG91dF9yNV9CIiksCiAgICAoIkNfcmwiLCBmIntQWX0gdHJhaW5fcG9uZGVyX3JsLnB5IC0tbW9kZWwge01PREVMfSAtLWFkYXB0ZXIgb3V0X3poICIKICAgICAgICAgICAgIGYiLS1oZWFkIG91dF9yNC9wb25kZXJfaGVhZC5zYWZldGVuc29ycyAtLWRhdGEge1RSQUlOfSAiCiAgICAgICAgICAgICBmIi0tbi1zYW1wbGVzIDk2IC0tYmF0Y2gtc2l6ZSA0IC0taXRlcnMgMjQgLS1zdGVwLXBlbmFsdHkgMC4zICIKICAgICAgICAgICAgIGYiLS1lbnQtY29lZiAwLjAxIC0tbHIgNWUtNCAtLW1heC1zdGVwcyA2IC0tbWF4LWxlbiA3NjggIgogICAgICAgICAgICAgZiItLWdyYWQtY2twdCAtLW91dHB1dCBvdXRfcjVfQyIpLApdCkVWQUxTID0gewogICAgIkExX21hcDgiOiAiLS1oZWFkIG91dF9yNV9BMS9wb25kZXJfaGVhZC5zYWZldGVuc29ycyAtLWFkYXB0ZXIgb3V0X3poIiwKICAgICJBMl93MTAiOiAiLS1oZWFkIG91dF9yNV9BMi9wb25kZXJfaGVhZC5zYWZldGVuc29ycyAtLWFkYXB0ZXIgb3V0X3poIiwKICAgICJBM19zaGFycCI6ICItLWhlYWQgb3V0X3I1X0EzL3BvbmRlcl9oZWFkLnNhZmV0ZW5zb3JzIC0tYWRhcHRlciBvdXRfemgiLAogICAgIkE0X21scCI6ICItLWhlYWQgb3V0X3I1X0E0L3BvbmRlcl9oZWFkLnNhZmV0ZW5zb3JzIC0tYWRhcHRlciBvdXRfemgiLAogICAgIkJfc3RlcGNlIjogIi0taGVhZCBvdXRfcjVfQi9wb25kZXJfaGVhZC5zYWZldGVuc29ycyAtLWFkYXB0ZXIgb3V0X3I1X0IiLAogICAgIkNfcmwiOiAiLS1oZWFkIG91dF9yNV9DL3BvbmRlcl9oZWFkLnNhZmV0ZW5zb3JzIC0tYWRhcHRlciBvdXRfemgiLAogICAgIlI0X2Jhc2VsaW5lIjogIi0taGVhZCBvdXRfcjQvcG9uZGVyX2hlYWQuc2FmZXRlbnNvcnMgLS1hZGFwdGVyIG91dF96aCIsCn0KCgpkZWYgZ3B1X3dhaXQobWluX2ZyZWVfbWI9OTAwMCwgdHJpZXM9MTIpOgogICAgIiIi5YWx5LqrIFQ0IOWuiOWNqzog562J5Yiw56m66Zey5pi+5a2Y5aSf55So5YaN5byA5LiL5LiA6Zi25q6144CCIiIiCiAgICBmb3IgXyBpbiByYW5nZSh0cmllcyk6CiAgICAgICAgciA9IHN1YnByb2Nlc3MucnVuKAogICAgICAgICAgICAibnZpZGlhLXNtaSAtLXF1ZXJ5LWdwdT1tZW1vcnkuZnJlZSAtLWZvcm1hdD1jc3Ysbm9oZWFkZXIsbm91bml0cyIsCiAgICAgICAgICAgIHNoZWxsPVRydWUsIGNhcHR1cmVfb3V0cHV0PVRydWUsIHRleHQ9VHJ1ZSkKICAgICAgICB0cnk6CiAgICAgICAgICAgIGZyZWUgPSBpbnQoci5zdGRvdXQuc3RyaXAoKS5zcGxpdGxpbmVzKClbMF0pCiAgICAgICAgZXhjZXB0IEV4Y2VwdGlvbjoKICAgICAgICAgICAgZnJlZSA9IG1pbl9mcmVlX21iCiAgICAgICAgaWYgZnJlZSA+PSBtaW5fZnJlZV9tYjoKICAgICAgICAgICAgcmV0dXJuIFRydWUKICAgICAgICBwcmludChmIltncHVfd2FpdF0g56m66ZeyIHtmcmVlfU1CIDwge21pbl9mcmVlX21ifU1CLCDnrYkgMTIwcyIsIGZsdXNoPVRydWUpCiAgICAgICAgdGltZS5zbGVlcCgxMjApCiAgICByZXR1cm4gRmFsc2UKCgpkZWYgc2goY21kLCB0aW1lb3V0PTcyMDApOgogICAgcHJpbnQoZiJcbiQge2NtZH1cbiIsIGZsdXNoPVRydWUpCiAgICByID0gc3VicHJvY2Vzcy5ydW4oY21kLCBzaGVsbD1UcnVlLCBjYXB0dXJlX291dHB1dD1UcnVlLCB0ZXh0PVRydWUsIHRpbWVvdXQ9dGltZW91dCkKICAgIHJldHVybiByLnJldHVybmNvZGUsIChyLnN0ZG91dCBvciAiIikgKyAoci5zdGRlcnIgb3IgIiIpCgoKZGVmIGxvZ19saW5lcyh0YWcsIHRleHQsIGtleXMsIGxpbWl0PTE4KToKICAgIGtlZXAgPSBbbCBmb3IgbCBpbiB0ZXh0LnNwbGl0bGluZXMoKSBpZiBhbnkoayBpbiBsIGZvciBrIGluIGtleXMpXQogICAgcHJpbnQoZiItLS0tLSBbe3RhZ31dIOWFs+mUruaXpeW/lyAtLS0tLSIsIGZsdXNoPVRydWUpCiAgICBmb3IgbCBpbiBrZWVwWy1saW1pdDpdOgogICAgICAgIHByaW50KGwsIGZsdXNoPVRydWUpCgoKZGVmIG1haW4oKToKICAgIHN1bW1hcnkgPSB7fQogICAgdDAgPSB0aW1lLnRpbWUoKQogICAgZm9yIG5hbWUsIGNtZCBpbiBTVEFHRVM6CiAgICAgICAgaGVhZCA9IGYib3V0X3I1X3tuYW1lLnNwbGl0KCdfJylbMF19IgogICAgICAgIGhlYWRfcGF0aCA9IG9zLnBhdGguam9pbihoZWFkLCAicG9uZGVyX2hlYWQuc2FmZXRlbnNvcnMiKQogICAgICAgIGR1bXBfcGF0aCA9IG9zLnBhdGguam9pbihoZWFkLCAiZXZhbF9kdW1wLmpzb25sIikKICAgICAgICBwcmludChmIlxuIyMjIyMjIyMjIyBTVEFHRSB7bmFtZX0gKHt0aW1lLnRpbWUoKS10MDouMGZ9cykgIyMjIyMjIyMjIyIsIGZsdXNoPVRydWUpCiAgICAgICAgZ3B1X3dhaXQoKQogICAgICAgICMgLS0tLSDorq3nu4MgLS0tLQogICAgICAgIGlmIG5vdCBvcy5wYXRoLmV4aXN0cyhoZWFkX3BhdGgpOgogICAgICAgICAgICB0cnk6CiAgICAgICAgICAgICAgICByYywgb3V0ID0gc2goY21kLCB0aW1lb3V0PTEwODAwKQogICAgICAgICAgICAgICAgbG9nX2xpbmVzKG5hbWUsIG91dCwgKCLmraXmlbDnoaznm5HnnaMiLCAi6YCQ5q2lIENFIiwgIuWPr+iuree7g+WPguaVsCIsICJzdGVwICIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIml0ZXIgIiwgIuWujOaIkDoiLCAi5bey5L+d5a2YIiwgIkVycm9yIiwgIlRyYWNlYmFjayIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIuacgOi/kTEw6L+t5LujIikpCiAgICAgICAgICAgICAgICBzdW1tYXJ5W25hbWVdID0gZiJ0cmFpbiByYz17cmN9IgogICAgICAgICAgICAgICAgaWYgcmMgIT0gMDoKICAgICAgICAgICAgICAgICAgICBzdW1tYXJ5W25hbWVdICs9ICIgKFRSQUlOX0ZBSUwsIOi3s+i/h+ivpemYtuautSBldmFsKSIKICAgICAgICAgICAgICAgICAgICBjb250aW51ZQogICAgICAgICAgICBleGNlcHQgRXhjZXB0aW9uIGFzIGU6CiAgICAgICAgICAgICAgICBzdW1tYXJ5W25hbWVdID0gZiJ0cmFpbiBFWEM9e2V9ICjot7Pov4cgZXZhbCkiCiAgICAgICAgICAgICAgICBjb250aW51ZQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIHByaW50KGYiW3NraXAtdHJhaW5dIHtoZWFkX3BhdGh9IOW3suWtmOWcqCIsIGZsdXNoPVRydWUpCiAgICAgICAgICAgIHN1bW1hcnlbbmFtZV0gPSAidHJhaW4gc2tpcHBlZCAoZXhpc3RzKSIKICAgICAgICAjIC0tLS0g6K+E5LywIC0tLS0KICAgICAgICBpZiBub3Qgb3MucGF0aC5leGlzdHMoZHVtcF9wYXRoKToKICAgICAgICAgICAgZWNmZyA9IEVWQUxTW25hbWVdCiAgICAgICAgICAgIGVjbWQgPSAoZiJ7UFl9IGV2YWxfcG9uZGVyLnB5IC0tbW9kZWwge01PREVMfSAtLWRhdGEge0VWQUx9IHtlY2ZnfSAiCiAgICAgICAgICAgICAgICAgICAgZiItLW1heC1zdGVwcyA2IC0tbWF4LWxlbiA3NjggLS1iYXRjaC1zaXplIDQgLS10YWcgcjVfe25hbWV9ICIKICAgICAgICAgICAgICAgICAgICBmIi0tZHVtcCB7ZHVtcF9wYXRofSIpCiAgICAgICAgICAgIHRyeToKICAgICAgICAgICAgICAgIHJjLCBvdXQgPSBzaChlY21kLCB0aW1lb3V0PTM2MDApCiAgICAgICAgICAgICAgICBsb2dfbGluZXMoZiJldmFsOntuYW1lfSIsIG91dCwgKCLlhajlsYAgQ0UiLCAi6Zq+5bqmIiwgImVhc3kiLCAibWVkaXVtIiwgImhhcmQiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAi6YCQ5qC35pysIiwgIkVycm9yIiwgIlRyYWNlYmFjayIpLCBsaW1pdD0xMikKICAgICAgICAgICAgICAgIHN1bW1hcnlbbmFtZV0gKz0gZiIgfCBldmFsIHJjPXtyY30iCiAgICAgICAgICAgIGV4Y2VwdCBFeGNlcHRpb24gYXMgZToKICAgICAgICAgICAgICAgIHN1bW1hcnlbbmFtZV0gKz0gZiIgfCBldmFsIEVYQz17ZX0iCiAgICAgICAgZWxzZToKICAgICAgICAgICAgc3VtbWFyeVtuYW1lXSArPSAiIHwgZXZhbCBza2lwcGVkIChleGlzdHMpIgoKICAgICMgLS0tLSBSNCDlkIzml6XlpI3or4Tln7rnur8gLS0tLQogICAgZHVtcF9wYXRoID0gIm91dF9yNC9ldmFsX2R1bXBfcjVkYXkuanNvbmwiCiAgICBpZiBub3Qgb3MucGF0aC5leGlzdHMoZHVtcF9wYXRoKToKICAgICAgICBwcmludChmIlxuIyMjIyMjIyMjIyBFVkFMIFI0X2Jhc2VsaW5lICh7dGltZS50aW1lKCktdDA6LjBmfXMpICMjIyMjIyMjIyMiLCBmbHVzaD1UcnVlKQogICAgICAgIGdwdV93YWl0KCkKICAgICAgICBlY21kID0gKGYie1BZfSBldmFsX3BvbmRlci5weSAtLW1vZGVsIHtNT0RFTH0gLS1kYXRhIHtFVkFMfSB7RVZBTFNbJ1I0X2Jhc2VsaW5lJ119ICIKICAgICAgICAgICAgICAgIGYiLS1tYXgtc3RlcHMgNiAtLW1heC1sZW4gNzY4IC0tYmF0Y2gtc2l6ZSA0IC0tdGFnIHI0X2Jhc2VsaW5lICIKICAgICAgICAgICAgICAgIGYiLS1kdW1wIHtkdW1wX3BhdGh9IikKICAgICAgICB0cnk6CiAgICAgICAgICAgIHJjLCBvdXQgPSBzaChlY21kLCB0aW1lb3V0PTM2MDApCiAgICAgICAgICAgIGxvZ19saW5lcygiZXZhbDpSNCIsIG91dCwgKCLlhajlsYAgQ0UiLCAi6Zq+5bqmIiwgImVhc3kiLCAibWVkaXVtIiwgImhhcmQiLCAi6YCQ5qC35pysIiksIDEwKQogICAgICAgICAgICBzdW1tYXJ5WyJSNF9iYXNlbGluZSJdID0gZiJldmFsIHJjPXtyY30iCiAgICAgICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBlOgogICAgICAgICAgICBzdW1tYXJ5WyJSNF9iYXNlbGluZSJdID0gZiJldmFsIEVYQz17ZX0iCgogICAgd2l0aCBvcGVuKCJyNV9zdW1tYXJ5Lmpzb24iLCAidyIsIGVuY29kaW5nPSJ1dGYtOCIpIGFzIGY6CiAgICAgICAganNvbi5kdW1wKHsid2FsbF9zIjogcm91bmQodGltZS50aW1lKCkgLSB0MCksICJzdGFnZXMiOiBzdW1tYXJ5fSwKICAgICAgICAgICAgICAgICAgZiwgZW5zdXJlX2FzY2lpPUZhbHNlLCBpbmRlbnQ9MSkKICAgIHByaW50KGYiXG4jIyMjIyMjIyMjIFNXRUVQIERPTkUgKHt0aW1lLnRpbWUoKS10MDouMGZ9cykgIyMjIyMjIyMjIyIsIGZsdXNoPVRydWUpCiAgICBmb3IgaywgdiBpbiBzdW1tYXJ5Lml0ZW1zKCk6CiAgICAgICAgcHJpbnQoZiIgIHtrfToge3Z9IiwgZmx1c2g9VHJ1ZSkKCgppZiBfX25hbWVfXyA9PSAiX19tYWluX18iOgogICAgbWFpbigpCg=="