| |
| """Stop PP-OCR training when held-out normalized edit similarity stalls or diverges.""" |
|
|
| from __future__ import annotations |
|
|
| import argparse |
| import os |
| import re |
| import signal |
| import subprocess |
| import time |
| from pathlib import Path |
|
|
|
|
| PATTERN = re.compile(r"cur metric,.*norm_edit_dis: ([0-9.eE+-]+)") |
|
|
|
|
| def main() -> None: |
| parser = argparse.ArgumentParser() |
| parser.add_argument("--log", type=Path, required=True) |
| parser.add_argument("--pid-file", type=Path, required=True) |
| parser.add_argument("--min-evals", type=int, default=5) |
| parser.add_argument("--patience", type=int, default=5) |
| parser.add_argument("--min-delta", type=float, default=0.002) |
| parser.add_argument("--divergence-delta", type=float, default=0.03) |
| parser.add_argument("--divergence-patience", type=int, default=2) |
| parser.add_argument("--initial-best", type=float, default=-1.0) |
| parser.add_argument("--repo-id") |
| parser.add_argument("--checkpoint-prefix", type=Path) |
| parser.add_argument("--config", type=Path) |
| parser.add_argument("--dictionary", type=Path) |
| parser.add_argument("--uploader", type=Path) |
| args = parser.parse_args() |
|
|
| best = args.initial_best |
| stale = diverged = seen = 0 |
| while True: |
| if not args.pid_file.exists(): |
| time.sleep(2) |
| continue |
| pid = int(args.pid_file.read_text()) |
| try: |
| os.kill(pid, 0) |
| except ProcessLookupError: |
| return |
|
|
| scores = [float(value) for value in PATTERN.findall(args.log.read_text(errors="ignore"))] |
| for score in scores[seen:]: |
| seen += 1 |
| improved = score > best + args.min_delta |
| if improved: |
| best, stale = score, 0 |
| else: |
| stale += 1 |
| diverged = diverged + 1 if score < best - args.divergence_delta else 0 |
| print( |
| f"eval={seen} score={score:.6f} best={best:.6f} " |
| f"stale={stale} diverged={diverged}", |
| flush=True, |
| ) |
| upload_values = ( |
| args.repo_id, |
| args.checkpoint_prefix, |
| args.config, |
| args.dictionary, |
| args.uploader, |
| ) |
| if improved and all(upload_values): |
| time.sleep(5) |
| try: |
| subprocess.run( |
| [ |
| str(args.uploader), |
| "--repo-id", args.repo_id, |
| "--checkpoint-prefix", str(args.checkpoint_prefix), |
| "--tag", f"eval-{seen:05d}", |
| "--metric", str(score), |
| "--config", str(args.config), |
| "--dictionary", str(args.dictionary), |
| ], |
| check=True, |
| ) |
| except Exception as error: |
| print(f"UPLOAD_FAILED {error}", flush=True) |
| if seen >= args.min_evals and ( |
| stale >= args.patience or diverged >= args.divergence_patience |
| ): |
| os.kill(pid, signal.SIGTERM) |
| print("EARLY_STOP", flush=True) |
| return |
| time.sleep(3) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
