UploadFile / hf_worker.go
ddosvn's picture
Deploy Upload ThTuanIT Full-stack (Go + Next.js + Python hf_xet + MongoDB)
7c5da52 verified
Raw History Blame Contribute Delete
6.29 kB
package main
import (
"bufio"
"encoding/json"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
)
const pyWorkerScript = `
import sys, json, os
from huggingface_hub import HfApi
token = os.environ.get("HF_TOKEN", "")
api = HfApi(token=token)
for line in sys.stdin:
line = line.strip()
if not line:
continue
try:
req = json.loads(line)
op = req.get("op")
bucket = req.get("bucket")
if op == "ping":
import huggingface_hub
res = {"ok": True, "version": huggingface_hub.__version__}
elif op == "ls":
prefix = req.get("prefix") or None
items = []
for entry in api.list_bucket_tree(bucket, prefix=prefix, recursive=True):
if getattr(entry, "type", "") == "file":
dt = getattr(entry, "uploaded_at", None)
updated = dt.strftime("%Y-%m-%d %H:%M:%S") if dt else ""
items.append({
"path": entry.path,
"size": getattr(entry, "size", 0) or 0,
"updated_at": updated,
"xet_hash": getattr(entry, "xet_hash", "") or ""
})
res = {"ok": True, "items": items}
elif op == "upload":
pairs = [(x["local"], x["remote"]) for x in req.get("files", [])]
api.batch_bucket_files(bucket, add=pairs)
res = {"ok": True}
elif op == "download":
api.download_bucket_files(bucket, [(req["remote"], req["local"])], raise_on_missing_files=True)
res = {"ok": True}
elif op == "rm":
api.batch_bucket_files(bucket, delete=[req["remote"]])
res = {"ok": True}
elif op == "mv":
old_p = req["old_path"]
new_p = req["new_path"]
infos = list(api.get_bucket_paths_info(bucket, [old_p]))
if not infos:
raise RuntimeError(f"File not found: {old_p}")
xhash = infos[0].xet_hash
api.batch_bucket_files(
bucket,
copy=[("bucket", bucket, xhash, new_p)],
delete=[old_p]
)
res = {"ok": True, "xet_hash": xhash}
else:
res = {"ok": False, "error": f"unknown op: {op}"}
except Exception as e:
res = {"ok": False, "error": str(e)}
sys.stdout.write(json.dumps(res) + "\n")
sys.stdout.flush()
`
// HFWorkerPool separates fast metadata ops (ls, mv, rm, ping) from heavy data transfers (upload, download)
// using channel-based free-worker queues so large uploads NEVER block /health or /api/files.
type HFWorkerPool struct {
metaQueue chan *hfWorker
xferQueue chan *hfWorker
version string
}
type hfWorker struct {
mu sync.Mutex
pyBin string
hfToken string
cmd *exec.Cmd
stdin io.WriteCloser
stdout *bufio.Reader
}
func resolvePythonFromHFCLI(hfCLI string) string {
if p, err := exec.LookPath("python"); err == nil {
return p
}
if p, err := exec.LookPath(hfCLI); err == nil {
dir := filepath.Dir(p)
candidate := filepath.Join(filepath.Dir(dir), "python.exe")
if _, err := os.Stat(candidate); err == nil {
return candidate
}
candidateUnix := filepath.Join(dir, "python3")
if _, err := os.Stat(candidateUnix); err == nil {
return candidateUnix
}
}
return "python"
}
func NewHFWorkerPool(xferSize int, hfCLI, hfToken string) *HFWorkerPool {
if xferSize < 2 {
xferSize = 2
}
metaSize := 2
pyBin := resolvePythonFromHFCLI(hfCLI)
pool := &HFWorkerPool{
metaQueue: make(chan *hfWorker, metaSize),
xferQueue: make(chan *hfWorker, xferSize),
version: "2.4.0 (Turbo Xet)",
}
for i := 0; i < metaSize; i++ {
w := &hfWorker{pyBin: pyBin, hfToken: hfToken}
_ = w.ensureRunning()
if i == 0 {
if res, err := w.call(map[string]any{"op": "ping"}); err == nil {
if v, ok := res["version"].(string); ok && v != "" {
pool.version = v
}
}
}
pool.metaQueue <- w
}
for i := 0; i < xferSize; i++ {
w := &hfWorker{pyBin: pyBin, hfToken: hfToken}
_ = w.ensureRunning()
pool.xferQueue <- w
}
return pool
}
func (w *hfWorker) ensureRunning() error {
if w.cmd != nil && w.cmd.Process != nil && w.cmd.ProcessState == nil {
return nil
}
cmd := exec.Command(w.pyBin, "-u", "-c", pyWorkerScript)
cmd.Env = append(
os.Environ(),
"HF_TOKEN="+w.hfToken,
"HF_HUB_DISABLE_PROGRESS_BARS=1",
"HF_XET_HIGH_PERFORMANCE=1",
"PYTHONIOENCODING=utf-8",
)
stdin, err := cmd.StdinPipe()
if err != nil {
return err
}
stdoutPipe, err := cmd.StdoutPipe()
if err != nil {
return err
}
if err := cmd.Start(); err != nil {
return err
}
w.cmd = cmd
w.stdin = stdin
w.stdout = bufio.NewReaderSize(stdoutPipe, 1<<20)
return nil
}
func (w *hfWorker) call(req map[string]any) (map[string]any, error) {
w.mu.Lock()
defer w.mu.Unlock()
if err := w.ensureRunning(); err != nil {
return nil, fmt.Errorf("failed to start python xet worker: %w", err)
}
payload, err := json.Marshal(req)
if err != nil {
return nil, err
}
if _, err := w.stdin.Write(append(payload, '\n')); err != nil {
w.cmd = nil
if err2 := w.ensureRunning(); err2 != nil {
return nil, err
}
if _, err2 := w.stdin.Write(append(payload, '\n')); err2 != nil {
return nil, err2
}
}
line, err := w.stdout.ReadString('\n')
if err != nil {
w.cmd = nil
return nil, fmt.Errorf("worker read failed: %w", err)
}
var resp map[string]any
if err := json.Unmarshal([]byte(strings.TrimSpace(line)), &resp); err != nil {
return nil, fmt.Errorf("invalid worker json: %w", err)
}
if ok, _ := resp["ok"].(bool); !ok {
errMsg, _ := resp["error"].(string)
return resp, fmt.Errorf("%s", errMsg)
}
return resp, nil
}
func (p *HFWorkerPool) Call(req map[string]any) (map[string]any, error) {
op, _ := req["op"].(string)
if op == "ping" {
return map[string]any{"ok": true, "version": p.version}, nil
}
// Route heavy transfers (upload / download) to xferQueue, and metadata ops (ls / mv / rm) to metaQueue
if op == "upload" || op == "download" {
w := <-p.xferQueue
defer func() { p.xferQueue <- w }()
return w.call(req)
}
w := <-p.metaQueue
defer func() { p.metaQueue <- w }()
return w.call(req)
}