Spaces:
Running
Running
Download hf_worker.go from TwanAPI/UploadFile: direct link, hf CLI and curl.
- Browser
- Download file 6.29 kB
-
https://huggingface.co/spaces/TwanAPI/UploadFile/resolve/main/hf_worker.go
- Command line
-
hf download hf://spaces/TwanAPI/UploadFile/hf_worker.go
-
curl -L -o hf_worker.go https://huggingface.co/spaces/TwanAPI/UploadFile/resolve/main/hf_worker.go
6.29 kB
| package main | |
| import ( | |
| "bufio" | |
| "encoding/json" | |
| "fmt" | |
| "io" | |
| "os" | |
| "os/exec" | |
| "path/filepath" | |
| "strings" | |
| "sync" | |
| ) | |
| const pyWorkerScript = ` | |
| import sys, json, os | |
| from huggingface_hub import HfApi | |
| token = os.environ.get("HF_TOKEN", "") | |
| api = HfApi(token=token) | |
| for line in sys.stdin: | |
| line = line.strip() | |
| if not line: | |
| continue | |
| try: | |
| req = json.loads(line) | |
| op = req.get("op") | |
| bucket = req.get("bucket") | |
| if op == "ping": | |
| import huggingface_hub | |
| res = {"ok": True, "version": huggingface_hub.__version__} | |
| elif op == "ls": | |
| prefix = req.get("prefix") or None | |
| items = [] | |
| for entry in api.list_bucket_tree(bucket, prefix=prefix, recursive=True): | |
| if getattr(entry, "type", "") == "file": | |
| dt = getattr(entry, "uploaded_at", None) | |
| updated = dt.strftime("%Y-%m-%d %H:%M:%S") if dt else "" | |
| items.append({ | |
| "path": entry.path, | |
| "size": getattr(entry, "size", 0) or 0, | |
| "updated_at": updated, | |
| "xet_hash": getattr(entry, "xet_hash", "") or "" | |
| }) | |
| res = {"ok": True, "items": items} | |
| elif op == "upload": | |
| pairs = [(x["local"], x["remote"]) for x in req.get("files", [])] | |
| api.batch_bucket_files(bucket, add=pairs) | |
| res = {"ok": True} | |
| elif op == "download": | |
| api.download_bucket_files(bucket, [(req["remote"], req["local"])], raise_on_missing_files=True) | |
| res = {"ok": True} | |
| elif op == "rm": | |
| api.batch_bucket_files(bucket, delete=[req["remote"]]) | |
| res = {"ok": True} | |
| elif op == "mv": | |
| old_p = req["old_path"] | |
| new_p = req["new_path"] | |
| infos = list(api.get_bucket_paths_info(bucket, [old_p])) | |
| if not infos: | |
| raise RuntimeError(f"File not found: {old_p}") | |
| xhash = infos[0].xet_hash | |
| api.batch_bucket_files( | |
| bucket, | |
| copy=[("bucket", bucket, xhash, new_p)], | |
| delete=[old_p] | |
| ) | |
| res = {"ok": True, "xet_hash": xhash} | |
| else: | |
| res = {"ok": False, "error": f"unknown op: {op}"} | |
| except Exception as e: | |
| res = {"ok": False, "error": str(e)} | |
| sys.stdout.write(json.dumps(res) + "\n") | |
| sys.stdout.flush() | |
| ` | |
| // HFWorkerPool separates fast metadata ops (ls, mv, rm, ping) from heavy data transfers (upload, download) | |
| // using channel-based free-worker queues so large uploads NEVER block /health or /api/files. | |
| type HFWorkerPool struct { | |
| metaQueue chan *hfWorker | |
| xferQueue chan *hfWorker | |
| version string | |
| } | |
| type hfWorker struct { | |
| mu sync.Mutex | |
| pyBin string | |
| hfToken string | |
| cmd *exec.Cmd | |
| stdin io.WriteCloser | |
| stdout *bufio.Reader | |
| } | |
| func resolvePythonFromHFCLI(hfCLI string) string { | |
| if p, err := exec.LookPath("python"); err == nil { | |
| return p | |
| } | |
| if p, err := exec.LookPath(hfCLI); err == nil { | |
| dir := filepath.Dir(p) | |
| candidate := filepath.Join(filepath.Dir(dir), "python.exe") | |
| if _, err := os.Stat(candidate); err == nil { | |
| return candidate | |
| } | |
| candidateUnix := filepath.Join(dir, "python3") | |
| if _, err := os.Stat(candidateUnix); err == nil { | |
| return candidateUnix | |
| } | |
| } | |
| return "python" | |
| } | |
| func NewHFWorkerPool(xferSize int, hfCLI, hfToken string) *HFWorkerPool { | |
| if xferSize < 2 { | |
| xferSize = 2 | |
| } | |
| metaSize := 2 | |
| pyBin := resolvePythonFromHFCLI(hfCLI) | |
| pool := &HFWorkerPool{ | |
| metaQueue: make(chan *hfWorker, metaSize), | |
| xferQueue: make(chan *hfWorker, xferSize), | |
| version: "2.4.0 (Turbo Xet)", | |
| } | |
| for i := 0; i < metaSize; i++ { | |
| w := &hfWorker{pyBin: pyBin, hfToken: hfToken} | |
| _ = w.ensureRunning() | |
| if i == 0 { | |
| if res, err := w.call(map[string]any{"op": "ping"}); err == nil { | |
| if v, ok := res["version"].(string); ok && v != "" { | |
| pool.version = v | |
| } | |
| } | |
| } | |
| pool.metaQueue <- w | |
| } | |
| for i := 0; i < xferSize; i++ { | |
| w := &hfWorker{pyBin: pyBin, hfToken: hfToken} | |
| _ = w.ensureRunning() | |
| pool.xferQueue <- w | |
| } | |
| return pool | |
| } | |
| func (w *hfWorker) ensureRunning() error { | |
| if w.cmd != nil && w.cmd.Process != nil && w.cmd.ProcessState == nil { | |
| return nil | |
| } | |
| cmd := exec.Command(w.pyBin, "-u", "-c", pyWorkerScript) | |
| cmd.Env = append( | |
| os.Environ(), | |
| "HF_TOKEN="+w.hfToken, | |
| "HF_HUB_DISABLE_PROGRESS_BARS=1", | |
| "HF_XET_HIGH_PERFORMANCE=1", | |
| "PYTHONIOENCODING=utf-8", | |
| ) | |
| stdin, err := cmd.StdinPipe() | |
| if err != nil { | |
| return err | |
| } | |
| stdoutPipe, err := cmd.StdoutPipe() | |
| if err != nil { | |
| return err | |
| } | |
| if err := cmd.Start(); err != nil { | |
| return err | |
| } | |
| w.cmd = cmd | |
| w.stdin = stdin | |
| w.stdout = bufio.NewReaderSize(stdoutPipe, 1<<20) | |
| return nil | |
| } | |
| func (w *hfWorker) call(req map[string]any) (map[string]any, error) { | |
| w.mu.Lock() | |
| defer w.mu.Unlock() | |
| if err := w.ensureRunning(); err != nil { | |
| return nil, fmt.Errorf("failed to start python xet worker: %w", err) | |
| } | |
| payload, err := json.Marshal(req) | |
| if err != nil { | |
| return nil, err | |
| } | |
| if _, err := w.stdin.Write(append(payload, '\n')); err != nil { | |
| w.cmd = nil | |
| if err2 := w.ensureRunning(); err2 != nil { | |
| return nil, err | |
| } | |
| if _, err2 := w.stdin.Write(append(payload, '\n')); err2 != nil { | |
| return nil, err2 | |
| } | |
| } | |
| line, err := w.stdout.ReadString('\n') | |
| if err != nil { | |
| w.cmd = nil | |
| return nil, fmt.Errorf("worker read failed: %w", err) | |
| } | |
| var resp map[string]any | |
| if err := json.Unmarshal([]byte(strings.TrimSpace(line)), &resp); err != nil { | |
| return nil, fmt.Errorf("invalid worker json: %w", err) | |
| } | |
| if ok, _ := resp["ok"].(bool); !ok { | |
| errMsg, _ := resp["error"].(string) | |
| return resp, fmt.Errorf("%s", errMsg) | |
| } | |
| return resp, nil | |
| } | |
| func (p *HFWorkerPool) Call(req map[string]any) (map[string]any, error) { | |
| op, _ := req["op"].(string) | |
| if op == "ping" { | |
| return map[string]any{"ok": true, "version": p.version}, nil | |
| } | |
| // Route heavy transfers (upload / download) to xferQueue, and metadata ops (ls / mv / rm) to metaQueue | |
| if op == "upload" || op == "download" { | |
| w := <-p.xferQueue | |
| defer func() { p.xferQueue <- w }() | |
| return w.call(req) | |
| } | |
| w := <-p.metaQueue | |
| defer func() { p.metaQueue <- w }() | |
| return w.call(req) | |
| } | |