package main import ( "bufio" "encoding/json" "fmt" "io" "os" "os/exec" "path/filepath" "strings" "sync" ) const pyWorkerScript = ` import sys, json, os from huggingface_hub import HfApi token = os.environ.get("HF_TOKEN", "") api = HfApi(token=token) for line in sys.stdin: line = line.strip() if not line: continue try: req = json.loads(line) op = req.get("op") bucket = req.get("bucket") if op == "ping": import huggingface_hub res = {"ok": True, "version": huggingface_hub.__version__} elif op == "ls": prefix = req.get("prefix") or None items = [] for entry in api.list_bucket_tree(bucket, prefix=prefix, recursive=True): if getattr(entry, "type", "") == "file": dt = getattr(entry, "uploaded_at", None) updated = dt.strftime("%Y-%m-%d %H:%M:%S") if dt else "" items.append({ "path": entry.path, "size": getattr(entry, "size", 0) or 0, "updated_at": updated, "xet_hash": getattr(entry, "xet_hash", "") or "" }) res = {"ok": True, "items": items} elif op == "upload": pairs = [(x["local"], x["remote"]) for x in req.get("files", [])] api.batch_bucket_files(bucket, add=pairs) res = {"ok": True} elif op == "download": api.download_bucket_files(bucket, [(req["remote"], req["local"])], raise_on_missing_files=True) res = {"ok": True} elif op == "rm": api.batch_bucket_files(bucket, delete=[req["remote"]]) res = {"ok": True} elif op == "mv": old_p = req["old_path"] new_p = req["new_path"] infos = list(api.get_bucket_paths_info(bucket, [old_p])) if not infos: raise RuntimeError(f"File not found: {old_p}") xhash = infos[0].xet_hash api.batch_bucket_files( bucket, copy=[("bucket", bucket, xhash, new_p)], delete=[old_p] ) res = {"ok": True, "xet_hash": xhash} else: res = {"ok": False, "error": f"unknown op: {op}"} except Exception as e: res = {"ok": False, "error": str(e)} sys.stdout.write(json.dumps(res) + "\n") sys.stdout.flush() ` // HFWorkerPool separates fast metadata ops (ls, mv, rm, ping) from heavy data transfers (upload, download) // using channel-based free-worker queues so large uploads NEVER block /health or /api/files. type HFWorkerPool struct { metaQueue chan *hfWorker xferQueue chan *hfWorker version string } type hfWorker struct { mu sync.Mutex pyBin string hfToken string cmd *exec.Cmd stdin io.WriteCloser stdout *bufio.Reader } func resolvePythonFromHFCLI(hfCLI string) string { if p, err := exec.LookPath("python"); err == nil { return p } if p, err := exec.LookPath(hfCLI); err == nil { dir := filepath.Dir(p) candidate := filepath.Join(filepath.Dir(dir), "python.exe") if _, err := os.Stat(candidate); err == nil { return candidate } candidateUnix := filepath.Join(dir, "python3") if _, err := os.Stat(candidateUnix); err == nil { return candidateUnix } } return "python" } func NewHFWorkerPool(xferSize int, hfCLI, hfToken string) *HFWorkerPool { if xferSize < 2 { xferSize = 2 } metaSize := 2 pyBin := resolvePythonFromHFCLI(hfCLI) pool := &HFWorkerPool{ metaQueue: make(chan *hfWorker, metaSize), xferQueue: make(chan *hfWorker, xferSize), version: "2.4.0 (Turbo Xet)", } for i := 0; i < metaSize; i++ { w := &hfWorker{pyBin: pyBin, hfToken: hfToken} _ = w.ensureRunning() if i == 0 { if res, err := w.call(map[string]any{"op": "ping"}); err == nil { if v, ok := res["version"].(string); ok && v != "" { pool.version = v } } } pool.metaQueue <- w } for i := 0; i < xferSize; i++ { w := &hfWorker{pyBin: pyBin, hfToken: hfToken} _ = w.ensureRunning() pool.xferQueue <- w } return pool } func (w *hfWorker) ensureRunning() error { if w.cmd != nil && w.cmd.Process != nil && w.cmd.ProcessState == nil { return nil } cmd := exec.Command(w.pyBin, "-u", "-c", pyWorkerScript) cmd.Env = append( os.Environ(), "HF_TOKEN="+w.hfToken, "HF_HUB_DISABLE_PROGRESS_BARS=1", "HF_XET_HIGH_PERFORMANCE=1", "PYTHONIOENCODING=utf-8", ) stdin, err := cmd.StdinPipe() if err != nil { return err } stdoutPipe, err := cmd.StdoutPipe() if err != nil { return err } if err := cmd.Start(); err != nil { return err } w.cmd = cmd w.stdin = stdin w.stdout = bufio.NewReaderSize(stdoutPipe, 1<<20) return nil } func (w *hfWorker) call(req map[string]any) (map[string]any, error) { w.mu.Lock() defer w.mu.Unlock() if err := w.ensureRunning(); err != nil { return nil, fmt.Errorf("failed to start python xet worker: %w", err) } payload, err := json.Marshal(req) if err != nil { return nil, err } if _, err := w.stdin.Write(append(payload, '\n')); err != nil { w.cmd = nil if err2 := w.ensureRunning(); err2 != nil { return nil, err } if _, err2 := w.stdin.Write(append(payload, '\n')); err2 != nil { return nil, err2 } } line, err := w.stdout.ReadString('\n') if err != nil { w.cmd = nil return nil, fmt.Errorf("worker read failed: %w", err) } var resp map[string]any if err := json.Unmarshal([]byte(strings.TrimSpace(line)), &resp); err != nil { return nil, fmt.Errorf("invalid worker json: %w", err) } if ok, _ := resp["ok"].(bool); !ok { errMsg, _ := resp["error"].(string) return resp, fmt.Errorf("%s", errMsg) } return resp, nil } func (p *HFWorkerPool) Call(req map[string]any) (map[string]any, error) { op, _ := req["op"].(string) if op == "ping" { return map[string]any{"ok": true, "version": p.version}, nil } // Route heavy transfers (upload / download) to xferQueue, and metadata ops (ls / mv / rm) to metaQueue if op == "upload" || op == "download" { w := <-p.xferQueue defer func() { p.xferQueue <- w }() return w.call(req) } w := <-p.metaQueue defer func() { p.metaQueue <- w }() return w.call(req) }