File size: 1,687 Bytes
057ec4b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 | """Re-run manifest for the 10 framework PanNuke fold02/03 jobs that died on a
transient GPU4 uncorrectable-ECC burst. Excludes attention_unet (those 2 are
finishing in the original queue). Lower concurrency (jobs_per_gpu=2)."""
import json, glob, re, os
base = {}
for p in sorted(glob.glob(os.path.expanduser("~/.aris_queue/runs/20260605T13*/manifest.json"))):
d = json.load(open(p)); js = d.get("jobs") or d.get("phases", [{}])[0].get("jobs", [])
for j in js:
c = j.get("cmd", "")
if "pannuke" in c and "--protocol fold01" in c and "--seed 0" in c:
m = re.search(r"--arch (\S+)", c)
if m:
base.setdefault(m.group(1), c)
archs = ["unet", "unetpp", "deeplabv3plus", "transunet", "swinunet"] # attention_unet excluded
jobs = []
for proto in ["fold02", "fold03"]:
for a in archs:
c = base[a].replace("--protocol fold01", "--protocol " + proto)
out = "results/baselines/pannuke_semantic_%s/%s/seed0/metrics.json" % (proto, a)
jobs.append({"id": "fw_%s_%s" % (proto, a), "cmd": c, "expected_output": out})
manifest = {"project": "pannuke_fw_rerun", "cwd": "/home/wzhang/LSC/Code/NPJ", "conda": "seggen",
"ssh": "a100", "gpus": [4, 5], "jobs_per_gpu": 2, "max_parallel": 4,
"gpu_free_threshold_mib": 60000, "oom_retry": {"delay": 240, "max_attempts": 3},
"phases": [{"name": "fw", "depends_on": [], "jobs": jobs}]}
RUN = "pannuke_fwre_" + os.environ["TS"]
rd = os.path.expanduser("~/.aris_queue/runs/" + RUN)
os.makedirs(rd + "/logs", exist_ok=True)
json.dump(manifest, open(rd + "/manifest.json", "w"), indent=2)
print("RUN=" + RUN, "jobs=" + str(len(jobs)))
|