Add portable Stage 6 native-bbox recovery bundle

training-code/stage6-native-v2/README.md

@@ -0,0 +1,50 @@
+# Bina 0.2 Stage 6 portable recovery bundle
+
+This bundle resumes the native-bbox corrective lane on an intact or replacement
+eight-GPU cluster.
+
+## Immutable inputs
+
+- Hugging Face artifact:
+  `Reza2kn/Bina-0.2-RizehPizeh/training-data/native-replay-v1.tar`
+- Archive SHA-256:
+  `2cb4b0507a1f6fc1fcf6619b679de5fd35a52e2ee8f55c57ce5ebf00c52ffc40`
+- Accepted replay:
+  7,041 hash-matched pages and 206,917 samples
+- Fixed compositor:
+  6,984 pages and 205,289 samples
+- Archived compositor fallback:
+  57 pages and 1,628 samples
+- Excluded:
+  485 pages that matched neither known production compositor
+
+## Cluster prerequisites
+
+The cluster root is `/workspace/bina02-ppocr` and must contain:
+
+- `PaddleOCR/`
+- `venv/`
+- `bina02_stage5_hardforms.yml`
+- `watch_bina02_stage4_eval.py`
+- `run/train-stage4-clean2m-lr3e6/best_accuracy`
+- `data/corpus-v4/balanced`
+
+The existing RTL label-encoding patch must remain active in PaddleOCR.
+
+## Recovery sequence
+
+1. Copy the three executable files in this directory to
+   `/workspace/bina02-ppocr`.
+2. Obtain a fresh final CDN URL for the Hugging Face replay archive.
+3. Run `fetch_native_replay_v2.sh "$SIGNED_CDN_URL"` under supervisor.
+4. Run `run_bina02_stage6_v2.sh` under supervisor.
+5. Export the best accepted checkpoint.
+6. Run the exact cached-Surya Persian OCR Triple Threat benchmark.
+7. Publish only if printed S3 and handwriting S3 are both below 10 with zero
+   missing predictions.
+
+The training mix is approximately 30 percent exact native forms, 35 percent
+general handwriting replay, and 35 percent printed replay. It uses a 1e-6
+learning rate, frozen batch normalization, trainable backbone blocks 4 and 5
+plus CTC layers, aspect-ratio-preserving resize and padding, and no geometric
+`RecAug`.

training-code/stage6-native-v2/build_bina02_stage6_native_mix_v2.py

@@ -0,0 +1,290 @@
+from __future__ import annotations
+
+import copy
+import hashlib
+import json
+import os
+from pathlib import Path
+
+import lmdb
+import yaml
+
+ROOT = Path("/workspace/bina02-ppocr")
+TRANSFER = Path(
+    os.environ.get(
+        "BINA_NATIVE_REPLAY_TRANSFER",
+        ROOT / "data/corpus-v5-native-replay/replay-transfer-v2",
+    )
+)
+FIXED_ROOT = TRANSFER / "full-fixed-v2"
+STALE_ROOT = TRANSFER / "full-stale-fallback-v1"
+FIXED = FIXED_ROOT / "lmdb"
+STALE = STALE_ROOT / "lmdb"
+BALANCED = ROOT / "data/corpus-v4/balanced"
+MIX = ROOT / "data/corpus-v5-native-replay/stage6-mix-v2"
+CONFIG = ROOT / "bina02_stage6_nativeforms_v2.yml"
+COMBINED_EVAL_CONFIG = ROOT / "bina02_stage6_combined_eval_stage4_v2.yml"
+NATIVE_EVAL_CONFIG = ROOT / "bina02_stage6_native_eval_stage4_v2.yml"
+RUN = ROOT / "run/train-stage6-nativeforms-v2-30-35-35-lr1e6"
+STAGE4 = ROOT / "run/train-stage4-clean2m-lr3e6/best_accuracy"
+
+EXPECTED_SUMMARIES = {
+    FIXED_ROOT: {
+        "pages": 6984,
+        "samples": 205289,
+        "train_samples": 198483,
+        "eval_samples": 6806,
+        "rejected": 0,
+        "mismatched": 542,
+    },
+    STALE_ROOT: {
+        "pages": 57,
+        "samples": 1628,
+        "train_samples": 1628,
+        "eval_samples": 0,
+        "rejected": 0,
+        "mismatched": 485,
+    },
+}
+
+
+def verify_summary(root: Path, expected: dict[str, int]) -> None:
+    summary_path = root / "SUMMARY.json"
+    if not summary_path.exists():
+        raise SystemExit(f"missing replay summary: {summary_path}")
+    summary = json.loads(summary_path.read_text())
+    actual = {key: summary.get(key) for key in expected}
+    if actual != expected:
+        raise SystemExit(
+            f"replay summary mismatch for {root}: expected={expected}, actual={actual}"
+        )
+
+
+def leaves(root: Path) -> list[Path]:
+    return sorted({path.parent for path in root.rglob("data.mdb")})
+
+
+def count(path: Path) -> int:
+    env = lmdb.open(
+        str(path),
+        readonly=True,
+        lock=False,
+        readahead=False,
+        max_readers=1,
+    )
+    with env.begin() as txn:
+        value = txn.get(b"num-samples")
+        result = int(value) if value else 0
+    env.close()
+    return result
+
+
+def stable(paths: list[Path], tag: str) -> list[Path]:
+    return sorted(
+        paths,
+        key=lambda path: hashlib.sha256((tag + str(path)).encode()).hexdigest(),
+    )
+
+
+def choose(paths: list[Path], target: int, tag: str) -> list[tuple[Path, int]]:
+    selected: list[tuple[Path, int]] = []
+    total = 0
+    for path in stable(paths, tag):
+        samples = count(path)
+        if samples <= 0:
+            continue
+        selected.append((path, samples))
+        total += samples
+        if total >= target:
+            break
+    if total < target:
+        raise SystemExit(f"{tag} has only {total} samples; target is {target}")
+    return selected
+
+
+def link_group(
+    group: str,
+    entries: list[tuple[Path, int]],
+) -> list[dict[str, object]]:
+    base = MIX / group
+    base.mkdir(parents=True, exist_ok=False)
+    rows: list[dict[str, object]] = []
+    for index, (source, samples) in enumerate(entries):
+        destination = base / f"{index:04d}-{source.parent.name}-{source.name}"
+        destination.mkdir()
+        os.link(source / "data.mdb", destination / "data.mdb")
+        rows.append(
+            {
+                "source": str(source),
+                "linked": str(destination),
+                "samples": samples,
+            }
+        )
+    return rows
+
+
+if MIX.exists():
+    raise SystemExit(f"refusing to overwrite existing mix: {MIX}")
+for replay_root, expected in EXPECTED_SUMMARIES.items():
+    verify_summary(replay_root, expected)
+for required in (FIXED, STALE, BALANCED, STAGE4):
+    if not required.exists():
+        raise SystemExit(f"missing required input: {required}")
+
+native_train = []
+for path in leaves(FIXED / "train") + leaves(STALE / "train"):
+    samples = count(path)
+    if samples > 0:
+        native_train.append((path, samples))
+
+native_train_total = sum(samples for _, samples in native_train)
+if native_train_total != 200111:
+    raise SystemExit(
+        f"native train count mismatch: expected 200111, got {native_train_total}"
+    )
+
+replay_target = (native_train_total * 35 + 29) // 30
+handwriting_replay = choose(
+    leaves(BALANCED / "train/00_handwriting"),
+    replay_target,
+    "stage6-v2-handwriting",
+)
+printed_replay = choose(
+    leaves(BALANCED / "train/01_printed"),
+    replay_target,
+    "stage6-v2-printed",
+)
+
+train_rows = {
+    "native_forms": link_group("train/00_nativeforms", native_train),
+    "handwriting_replay": link_group(
+        "train/01_handwriting_replay",
+        handwriting_replay,
+    ),
+    "printed_replay": link_group(
+        "train/02_printed_replay",
+        printed_replay,
+    ),
+}
+
+native_eval = []
+for path in leaves(FIXED / "eval") + leaves(STALE / "eval"):
+    samples = count(path)
+    if samples > 0:
+        native_eval.append((path, samples))
+handwriting_eval = [
+    (path, count(path))
+    for path in leaves(BALANCED / "eval/00_handwriting")
+    if count(path) > 0
+]
+printed_eval = [
+    (path, count(path))
+    for path in leaves(BALANCED / "eval/01_printed")
+    if count(path) > 0
+]
+
+eval_rows = {
+    "native_forms": link_group("eval/00_nativeforms", native_eval),
+    "handwriting": link_group("eval/01_handwriting", handwriting_eval),
+    "printed": link_group("eval/02_printed", printed_eval),
+}
+
+train_counts = {
+    key: sum(row["samples"] for row in rows)
+    for key, rows in train_rows.items()
+}
+eval_counts = {
+    key: sum(row["samples"] for row in rows)
+    for key, rows in eval_rows.items()
+}
+if eval_counts["native_forms"] != 6806:
+    raise SystemExit(
+        f"native eval count mismatch: expected 6806, got {eval_counts['native_forms']}"
+    )
+
+train_total = sum(train_counts.values())
+handwriting_domain_rows = (
+    eval_counts["native_forms"] + eval_counts["handwriting"]
+)
+manifest = {
+    "schema": "bina02.stage6-native-mix.v2",
+    "replay_archive_sha256": (
+        "2cb4b0507a1f6fc1fcf6619b679de5fd35a52e2ee8f55c57ce5ebf00c52ffc40"
+    ),
+    "train": train_rows,
+    "eval": eval_rows,
+    "train_counts": train_counts,
+    "eval_counts": eval_counts,
+    "train_total": train_total,
+    "train_proportions": {
+        key: value / train_total for key, value in train_counts.items()
+    },
+    "handwriting_domain_rows": handwriting_domain_rows,
+    "printed_domain_rows": eval_counts["printed"],
+    "preprocessing": (
+        "native bbox crops with context; aspect-ratio-preserving resize+padding; "
+        "RecAug disabled"
+    ),
+}
+(MIX / "MIX_MANIFEST.json").write_text(
+    json.dumps(manifest, ensure_ascii=False, indent=2) + "\n"
+)
+
+base = yaml.safe_load((ROOT / "bina02_stage5_hardforms.yml").read_text())
+base["Global"]["epoch_num"] = 2
+base["Global"]["save_model_dir"] = str(RUN)
+base["Global"]["pretrained_model"] = str(STAGE4)
+base["Global"]["checkpoints"] = None
+base["Global"]["eval_batch_step"] = [0, 200]
+base["Global"].pop("stage5_mix_manifest", None)
+base["Global"]["stage6_mix_manifest"] = str(MIX / "MIX_MANIFEST.json")
+base["Global"]["eval_handwriting_rows"] = handwriting_domain_rows
+base["Global"]["eval_printed_rows"] = eval_counts["printed"]
+base["Optimizer"]["lr"]["learning_rate"] = 1.0e-6
+base["Metric"]["domain_split_index"] = handwriting_domain_rows
+base["Train"]["dataset"]["data_dir"] = str(MIX / "train")
+base["Train"]["dataset"]["transforms"] = [
+    transform
+    for transform in base["Train"]["dataset"]["transforms"]
+    if "RecAug" not in transform
+]
+base["Eval"]["dataset"]["data_dir"] = str(MIX / "eval")
+CONFIG.write_text(yaml.safe_dump(base, sort_keys=False, allow_unicode=True))
+
+combined_eval = copy.deepcopy(base)
+combined_eval["Global"]["pretrained_model"] = None
+combined_eval["Global"]["checkpoints"] = str(STAGE4)
+COMBINED_EVAL_CONFIG.write_text(
+    yaml.safe_dump(combined_eval, sort_keys=False, allow_unicode=True)
+)
+
+native_eval_config = copy.deepcopy(combined_eval)
+native_eval_config["Eval"]["dataset"]["data_dir"] = str(
+    MIX / "eval/00_nativeforms"
+)
+native_eval_config["Metric"] = {
+    "name": "RecMetric",
+    "main_indicator": "norm_edit_dis",
+    "ignore_space": False,
+}
+NATIVE_EVAL_CONFIG.write_text(
+    yaml.safe_dump(native_eval_config, sort_keys=False, allow_unicode=True)
+)
+
+print(
+    json.dumps(
+        {
+            "manifest": str(MIX / "MIX_MANIFEST.json"),
+            "config": str(CONFIG),
+            "combined_eval_config": str(COMBINED_EVAL_CONFIG),
+            "native_eval_config": str(NATIVE_EVAL_CONFIG),
+            "train_counts": train_counts,
+            "eval_counts": eval_counts,
+            "train_total": train_total,
+            "train_proportions": manifest["train_proportions"],
+            "handwriting_domain_rows": handwriting_domain_rows,
+        },
+        ensure_ascii=False,
+        indent=2,
+    )
+)

training-code/stage6-native-v2/fetch_native_replay_v2.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+set -euo pipefail
+
+ROOT=/workspace/bina02-ppocr/data/corpus-v5-native-replay
+EXPECTED_SHA256=2cb4b0507a1f6fc1fcf6619b679de5fd35a52e2ee8f55c57ce5ebf00c52ffc40
+URL=${1:?usage: fetch_native_replay_v2.sh SIGNED_CDN_URL}
+PARTIAL="$ROOT/native-replay-v1.tar.part"
+ARCHIVE="$ROOT/native-replay-v1.tar"
+DEST="$ROOT/replay-transfer-v2"
+
+if [ -e "$ARCHIVE" ] || [ -e "$DEST" ]; then
+  echo "final archive or extraction destination already exists; refusing to overwrite"
+  exit 3
+fi
+
+mkdir -p "$ROOT"
+resume=()
+if [ -s "$PARTIAL" ]; then
+  resume=(-C -)
+fi
+
+curl -4 -fL \
+  --connect-timeout 15 \
+  --retry 30 \
+  --retry-delay 5 \
+  --retry-all-errors \
+  "${resume[@]}" \
+  -o "$PARTIAL" \
+  "$URL"
+
+printf '%s  %s\n' "$EXPECTED_SHA256" "$PARTIAL" | sha256sum -c -
+mv "$PARTIAL" "$ARCHIVE"
+mkdir "$DEST"
+tar --sparse -xf "$ARCHIVE" -C "$DEST"
+printf '{"schema":"bina02.native-replay-fetch.v2","sha256":"%s","completed_at":"%s"}\n' \
+  "$EXPECTED_SHA256" \
+  "$(date -Iseconds)" \
+  > "$DEST/FETCH_COMPLETE.json"
+echo "$(date -Iseconds) FETCH_AND_EXTRACT_COMPLETE"

training-code/stage6-native-v2/run_bina02_stage6_v2.sh

@@ -0,0 +1,93 @@
+#!/bin/bash
+set -euo pipefail
+
+ROOT=/workspace/bina02-ppocr
+RUN="$ROOT/run/train-stage6-nativeforms-v2-30-35-35-lr1e6"
+CONFIG="$ROOT/bina02_stage6_nativeforms_v2.yml"
+COMBINED_EVAL_CONFIG="$ROOT/bina02_stage6_combined_eval_stage4_v2.yml"
+NATIVE_EVAL_CONFIG="$ROOT/bina02_stage6_native_eval_stage4_v2.yml"
+BUILDER="$ROOT/build_bina02_stage6_native_mix_v2.py"
+VENV="$ROOT/venv"
+
+if [ -e "$RUN/status.log" ]; then
+  echo "Stage 6 v2 status already exists; refusing to duplicate the run"
+  exit 3
+fi
+
+mkdir -p "$RUN" "$RUN/distributed-logs"
+"$VENV/bin/python" "$BUILDER" > "$RUN/mix-build.json"
+
+cd "$ROOT/PaddleOCR"
+export PYTHONPATH="$ROOT/PaddleOCR"
+export FLAGS_allocator_strategy=auto_growth
+export OMP_NUM_THREADS=4
+
+CUDA_VISIBLE_DEVICES=0 "$VENV/bin/python" tools/eval.py \
+  -c "$COMBINED_EVAL_CONFIG" \
+  > "$RUN/stage4-combined-baseline.log" 2>&1 &
+combined_eval_pid=$!
+
+CUDA_VISIBLE_DEVICES=1 "$VENV/bin/python" tools/eval.py \
+  -c "$NATIVE_EVAL_CONFIG" \
+  > "$RUN/stage4-native-baseline.log" 2>&1 &
+native_eval_pid=$!
+
+wait "$combined_eval_pid"
+wait "$native_eval_pid"
+
+read -r baseline_handwriting baseline_printed < <(
+  "$VENV/bin/python" - "$RUN/stage4-combined-baseline.log" <<'PY'
+import re
+import sys
+from pathlib import Path
+
+text = Path(sys.argv[1]).read_text(errors="replace")
+matches = re.findall(
+    r"cur metric,.*?handwriting_norm_edit_dis:\s*([0-9.eE+-]+)"
+    r".*?printed_norm_edit_dis:\s*([0-9.eE+-]+)",
+    text,
+)
+if not matches:
+    raise SystemExit("could not parse Stage 4 combined-domain baseline")
+print(*matches[-1])
+PY
+)
+
+printf 'BASELINE_HANDWRITING=%s\nBASELINE_PRINTED=%s\n' \
+  "$baseline_handwriting" \
+  "$baseline_printed" \
+  > "$RUN/baselines.env"
+
+cp "$CONFIG" "$RUN/launch-config.yml"
+printf '%s TRAIN_START\n' "$(date -Iseconds)" > "$RUN/status.log"
+
+setsid "$VENV/bin/python" -m paddle.distributed.launch \
+  --gpus "0,1,2,3,4,5,6,7" \
+  --log_dir "$RUN/distributed-logs" \
+  tools/train.py \
+  -c "$RUN/launch-config.yml" \
+  > "$RUN/train-launch.log" 2>&1 &
+trainer=$!
+printf '%s TRAIN_PID=%s\n' "$(date -Iseconds)" "$trainer" >> "$RUN/status.log"
+
+"$VENV/bin/python" "$ROOT/watch_bina02_stage4_eval.py" \
+  --log "$RUN/train-launch.log" \
+  --pgid "$trainer" \
+  --state "$RUN/eval-gate-state.json" \
+  --baseline-handwriting "$baseline_handwriting" \
+  --baseline-printed "$baseline_printed" \
+  --handwriting-margin 0.003 \
+  --printed-margin 0.002 \
+  --patience 2 \
+  --poll-seconds 5 \
+  > "$RUN/eval-gate.log" 2>&1 &
+watcher=$!
+
+set +e
+wait "$trainer"
+rc=$?
+set -e
+kill "$watcher" 2>/dev/null || true
+wait "$watcher" 2>/dev/null || true
+printf '%s TRAIN_EXIT_CODE=%s\n' "$(date -Iseconds)" "$rc" >> "$RUN/status.log"
+exit "$rc"