Spaces:
Sleeping
Sleeping
feat: implement resume functionality in batch feature extraction to skip processed samples
Browse files
app/training/extract_features_batch.py
CHANGED
|
@@ -232,11 +232,28 @@ def extract_batch(
|
|
| 232 |
failed = 0
|
| 233 |
t_start = _time.time()
|
| 234 |
|
| 235 |
-
|
| 236 |
-
|
| 237 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 238 |
writer = csv.DictWriter(f, fieldnames=out_columns)
|
| 239 |
-
|
|
|
|
| 240 |
f.flush()
|
| 241 |
|
| 242 |
with mp.Pool(processes=n_workers) as pool:
|
|
@@ -253,9 +270,9 @@ def extract_batch(
|
|
| 253 |
f.flush()
|
| 254 |
elapsed = _time.time() - t_start
|
| 255 |
rate = i / elapsed if elapsed > 0 else 0
|
| 256 |
-
eta = (
|
| 257 |
print(
|
| 258 |
-
f" [{i}/{
|
| 259 |
f"ok={success} fail={failed} "
|
| 260 |
f"rate={rate:.1f}/s eta={eta / 60:.1f}m",
|
| 261 |
flush=True,
|
|
|
|
| 232 |
failed = 0
|
| 233 |
t_start = _time.time()
|
| 234 |
|
| 235 |
+
done_paths: set[str] = set()
|
| 236 |
+
resume = output_path.exists() and output_path.stat().st_size > 0
|
| 237 |
+
if resume:
|
| 238 |
+
with open(output_path, "r", encoding="utf-8") as f_prev:
|
| 239 |
+
reader = csv.DictReader(f_prev)
|
| 240 |
+
for r in reader:
|
| 241 |
+
done_paths.add(r["file_path"])
|
| 242 |
+
print(f" Resuming: {len(done_paths)} samples already processed, skipping", flush=True)
|
| 243 |
+
|
| 244 |
+
tasks = [
|
| 245 |
+
(s["file_path"], int(s["label_int"]))
|
| 246 |
+
for s in samples
|
| 247 |
+
if s["file_path"] not in done_paths
|
| 248 |
+
]
|
| 249 |
+
total_remaining = len(tasks)
|
| 250 |
+
print(f" Remaining: {total_remaining} samples to process", flush=True)
|
| 251 |
+
|
| 252 |
+
file_mode = "a" if resume else "w"
|
| 253 |
+
with open(output_path, file_mode, newline="", encoding="utf-8") as f:
|
| 254 |
writer = csv.DictWriter(f, fieldnames=out_columns)
|
| 255 |
+
if not resume:
|
| 256 |
+
writer.writeheader()
|
| 257 |
f.flush()
|
| 258 |
|
| 259 |
with mp.Pool(processes=n_workers) as pool:
|
|
|
|
| 270 |
f.flush()
|
| 271 |
elapsed = _time.time() - t_start
|
| 272 |
rate = i / elapsed if elapsed > 0 else 0
|
| 273 |
+
eta = (total_remaining - i) / rate if rate > 0 else 0
|
| 274 |
print(
|
| 275 |
+
f" [{i}/{total_remaining}] "
|
| 276 |
f"ok={success} fail={failed} "
|
| 277 |
f"rate={rate:.1f}/s eta={eta / 60:.1f}m",
|
| 278 |
flush=True,
|