Add checkpoint uploads to eval monitor

training-kit/monitor_ppocr_eval.py

@@ -7,6 +7,7 @@ import argparse
 import os
 import re
 import signal
+import subprocess
 import time
 from pathlib import Path
 
@@ -24,6 +25,11 @@ def main() -> None:
     parser.add_argument("--divergence-delta", type=float, default=0.03)
     parser.add_argument("--divergence-patience", type=int, default=2)
     parser.add_argument("--initial-best", type=float, default=-1.0)
+    parser.add_argument("--repo-id")
+    parser.add_argument("--checkpoint-prefix", type=Path)
+    parser.add_argument("--config", type=Path)
+    parser.add_argument("--dictionary", type=Path)
+    parser.add_argument("--uploader", type=Path)
     args = parser.parse_args()
 
     best = args.initial_best
@@ -41,7 +47,8 @@ def main() -> None:
         scores = [float(value) for value in PATTERN.findall(args.log.read_text(errors="ignore"))]
         for score in scores[seen:]:
             seen += 1
-            if score > best + args.min_delta:
+            improved = score > best + args.min_delta
+            if improved:
                 best, stale = score, 0
             else:
                 stale += 1
@@ -51,6 +58,30 @@ def main() -> None:
                 f"stale={stale} diverged={diverged}",
                 flush=True,
             )
+            upload_values = (
+                args.repo_id,
+                args.checkpoint_prefix,
+                args.config,
+                args.dictionary,
+                args.uploader,
+            )
+            if improved and all(upload_values):
+                time.sleep(5)
+                try:
+                    subprocess.run(
+                        [
+                            str(args.uploader),
+                            "--repo-id", args.repo_id,
+                            "--checkpoint-prefix", str(args.checkpoint_prefix),
+                            "--tag", f"eval-{seen:05d}",
+                            "--metric", str(score),
+                            "--config", str(args.config),
+                            "--dictionary", str(args.dictionary),
+                        ],
+                        check=True,
+                    )
+                except Exception as error:
+                    print(f"UPLOAD_FAILED {error}", flush=True)
             if seen >= args.min_evals and (
                 stale >= args.patience or diverged >= args.divergence_patience
             ):