Add pinned borrowed-eight-hour campaign bootstrap

START_BORROWED_8H.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+set -euo pipefail
+SIMCT_BASE=/workspace/storage-shared/nlp/tungks
+SIMCT_HOST=tungdd11-sparse-vllm-core-0-0
+[[ "$(hostname)" == "$SIMCT_HOST" ]] || { echo 'WRONG_HOST: run only on the newly borrowed node'; exit 2; }
+SIMCT_POINTER="$SIMCT_BASE/borrow8-$SIMCT_HOST.launch"
+if [[ -e "$SIMCT_POINTER" ]]; then
+  echo 'ALLOCATION_ALREADY_RECORDED: inspect/recover this allocation; do not reset its clock'
+  cat "$SIMCT_POINTER"
+  exit 2
+fi
+SIMCT_STARTED="${SIMCT_BORROW_STARTED:-$(date +%s)}"
+[[ "$SIMCT_STARTED" =~ ^[0-9]+$ ]] || { echo 'START must be Unix integer seconds'; exit 2; }
+SIMCT_TASK="$(mktemp -d "$SIMCT_BASE/borrow8-XXXXXXXX")"
+(set -o noclobber; printf '%s\n%s\n' "$SIMCT_TASK" "$SIMCT_STARTED" > "$SIMCT_POINTER")
+printf 'ALLOCATION_ROOT=%s\nSTARTED=%s\n' "$SIMCT_TASK" "$SIMCT_STARTED"
+test -x /usr/bin/python3.12 || { echo 'MISSING_RUNTIME: /usr/bin/python3.12; keep the recorded allocation clock'; exit 2; }
+test -x /opt/venvs/simct-b200/bin/python || { echo 'MISSING_RUNTIME: /opt/venvs/simct-b200/bin/python; keep the recorded allocation clock'; exit 2; }
+SIMCT_REV=d849970059510104e763209685b80645af5f88d9
+SIMCT_URL="https://huggingface.co/codemaivanngu/simct/resolve/$SIMCT_REV"
+for SIMCT_FILE in simct-b200-portable.bundle job-manager-source.tar; do
+  curl --fail --location --silent --show-error --retry 3 --connect-timeout 20 \
+    --proxy http://10.30.154.118:80 --noproxy "" \
+    "$SIMCT_URL/$SIMCT_FILE" -o "$SIMCT_TASK/$SIMCT_FILE"
+done
+(
+  cd "$SIMCT_TASK"
+  printf '%s  %s\n' \
+    885f2fb7d994720f2a06e1a151493bb2b8c3ff23c8aaff1002202b02102df6a3 simct-b200-portable.bundle \
+    cdbda29a2d35fef8c3315a81ba4bf6d1cb7402b762cf8609379d0e60773c4914 job-manager-source.tar \
+    | sha256sum --check -
+)
+git clone --branch vdt/ops/b200-portable "$SIMCT_TASK/simct-b200-portable.bundle" "$SIMCT_TASK/source"
+[[ "$(git -C "$SIMCT_TASK/source" rev-parse HEAD)" == 6b9442ad463dc3781436c7cc94bb4e49d49c81d8 ]] || { echo 'SOURCE_MISMATCH'; exit 2; }
+/usr/bin/python3.12 - "$SIMCT_TASK" <<'PY'
+import sys,tarfile
+from pathlib import Path
+root=Path(sys.argv[1])
+with tarfile.open(root/'job-manager-source.tar') as archive:
+    for member in archive.getmembers():
+        if member.name!='job-manager' and not member.name.startswith('job-manager/'):
+            raise ValueError('Unexpected archive member')
+    archive.extractall(root,filter='data')
+PY
+bash "$SIMCT_TASK/source/experiments/runai/start_borrowed_campaign.sh" \
+  "$SIMCT_TASK/work" "$SIMCT_TASK/job-manager" "$SIMCT_STARTED" "$SIMCT_HOST"