File size: 3,353 Bytes
cb074c8 3456ea9 cb074c8 3456ea9 cb074c8 3456ea9 cb074c8 3456ea9 cb074c8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 | #!/usr/bin/env python3
"""Stop PP-OCR training when held-out normalized edit similarity stalls or diverges."""
from __future__ import annotations
import argparse
import os
import re
import signal
import subprocess
import time
from pathlib import Path
PATTERN = re.compile(r"cur metric,.*norm_edit_dis: ([0-9.eE+-]+)")
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--log", type=Path, required=True)
parser.add_argument("--pid-file", type=Path, required=True)
parser.add_argument("--min-evals", type=int, default=5)
parser.add_argument("--patience", type=int, default=5)
parser.add_argument("--min-delta", type=float, default=0.002)
parser.add_argument("--divergence-delta", type=float, default=0.03)
parser.add_argument("--divergence-patience", type=int, default=2)
parser.add_argument("--initial-best", type=float, default=-1.0)
parser.add_argument("--repo-id")
parser.add_argument("--checkpoint-prefix", type=Path)
parser.add_argument("--config", type=Path)
parser.add_argument("--dictionary", type=Path)
parser.add_argument("--uploader", type=Path)
args = parser.parse_args()
best = args.initial_best
stale = diverged = seen = 0
while True:
if not args.pid_file.exists():
time.sleep(2)
continue
pid = int(args.pid_file.read_text())
try:
os.kill(pid, 0)
except ProcessLookupError:
return
scores = [float(value) for value in PATTERN.findall(args.log.read_text(errors="ignore"))]
for score in scores[seen:]:
seen += 1
improved = score > best + args.min_delta
if improved:
best, stale = score, 0
else:
stale += 1
diverged = diverged + 1 if score < best - args.divergence_delta else 0
print(
f"eval={seen} score={score:.6f} best={best:.6f} "
f"stale={stale} diverged={diverged}",
flush=True,
)
upload_values = (
args.repo_id,
args.checkpoint_prefix,
args.config,
args.dictionary,
args.uploader,
)
if improved and all(upload_values):
time.sleep(5)
try:
subprocess.run(
[
str(args.uploader),
"--repo-id", args.repo_id,
"--checkpoint-prefix", str(args.checkpoint_prefix),
"--tag", f"eval-{seen:05d}",
"--metric", str(score),
"--config", str(args.config),
"--dictionary", str(args.dictionary),
],
check=True,
)
except Exception as error:
print(f"UPLOAD_FAILED {error}", flush=True)
if seen >= args.min_evals and (
stale >= args.patience or diverged >= args.divergence_patience
):
os.kill(pid, signal.SIGTERM)
print("EARLY_STOP", flush=True)
return
time.sleep(3)
if __name__ == "__main__":
main()
|