package main import ( "bufio" "context" "crypto/sha256" "embed" "encoding/hex" "encoding/json" "errors" "fmt" "io" "log" "mime" "net/http" "net/http/httputil" "net/url" "os" "os/exec" "path" "path/filepath" "strconv" "strings" "sync" "time" ) //go:embed web/index.html var webFS embed.FS const bucketID = "TwanAPI/ThTuan" type FileItem struct { Path string `json:"path"` Size int64 `json:"size"` UpdatedAt string `json:"updated_at,omitempty"` Owner string `json:"owner,omitempty"` } type Config struct { HFCLI string HFToken string APIKey string ListenAddr string TempDir string MaxUpload int64 CommandTime time.Duration } type App struct { cfg Config users *UserStore owners *FileOwnerStore shares *ShareStore mongo *MongoStore pool *HFWorkerPool cacheMu sync.Map // map[string]*sync.Mutex per cached file path listCacheMu sync.RWMutex listCache []FileItem listCacheTime time.Time pathAllocMu sync.Mutex reservedPaths map[string]bool chunkSessions sync.Map // map[string]*chunkUploadSession } type chunkUploadSession struct { id string owner string remotePath string tmpPath string file *os.File totalSize int64 createdAt time.Time mu sync.Mutex closed bool } type offsetWriter struct { f *os.File off int64 } func (ow *offsetWriter) Write(p []byte) (int, error) { n, err := ow.f.WriteAt(p, ow.off) ow.off += int64(n) return n, err } func (a *App) invalidateListCache() { a.listCacheMu.Lock() a.listCacheTime = time.Time{} a.listCacheMu.Unlock() } func (a *App) evictDownloadCache(remote string) { cacheDir := filepath.Join(a.cfg.TempDir, "hf-cache") hash := sha256.Sum256([]byte(remote)) ext := filepath.Ext(remote) cachedFile := filepath.Join(cacheDir, hex.EncodeToString(hash[:16])+ext) _ = os.Remove(cachedFile) } func (a *App) resolveUniqueRemotePath(scopedPath string) string { a.pathAllocMu.Lock() defer a.pathAllocMu.Unlock() if a.reservedPaths == nil { a.reservedPaths = make(map[string]bool) if a.pool != nil { if res, err := a.pool.Call(map[string]any{"op": "ls", "bucket": bucketID}); err == nil { if rawItems, ok := res["items"].([]any); ok { for _, ri := range rawItems { if m, ok := ri.(map[string]any); ok { if p, _ := m["path"].(string); p != "" { a.reservedPaths[p] = true } } } } } } } unique := nextAvailableRemotePath(scopedPath, func(candidate string) bool { if a.reservedPaths[candidate] { return true } if a.owners != nil && a.owners.HasPath(candidate) { return true } return false }) a.reservedPaths[unique] = true return unique } func (a *App) releaseReservedPath(remotePath string) { a.pathAllocMu.Lock() if a.reservedPaths != nil { delete(a.reservedPaths, remotePath) } a.pathAllocMu.Unlock() } func (a *App) fileLock(key string) *sync.Mutex { val, _ := a.cacheMu.LoadOrStore(key, &sync.Mutex{}) return val.(*sync.Mutex) } type contextKey string const userContextKey contextKey = "auth_user" type APIError struct { Error string `json:"error"` Detail string `json:"detail,omitempty"` } func main() { cfg, err := loadConfig() if err != nil { log.Fatal(err) } if err := os.MkdirAll(cfg.TempDir, 0700); err != nil { log.Fatal(err) } userStore, err := NewUserStore(env("USERS_FILE", "users.json"), cfg.APIKey) if err != nil { log.Fatalf("failed to load user store: %v", err) } ownerStore, err := NewFileOwnerStore(env("OWNERS_FILE", "file_owners.json")) if err != nil { log.Fatalf("failed to load file owner store: %v", err) } shareStore, err := NewShareStore(env("SHARES_FILE", "shares.json")) if err != nil { log.Fatalf("failed to load share store: %v", err) } var mongoStore *MongoStore if mongoURI := env("MONGODB_URI", os.Getenv("MONGO_URI")); mongoURI != "" { ms, mErr := NewMongoStore(mongoURI, env("MONGODB_DB", "upload_thtuanit")) if mErr != nil { log.Printf("WARNING: failed to connect to MongoDB (%v); falling back to local JSON store", mErr) } else { mongoStore = ms userStore.AttachMongo(mongoStore) ownerStore.AttachMongo(mongoStore) shareStore.AttachMongo(mongoStore) } } workerPool := NewHFWorkerPool(4, cfg.HFCLI, cfg.HFToken) app := &App{cfg: cfg, users: userStore, owners: ownerStore, shares: shareStore, mongo: mongoStore, pool: workerPool} mux := http.NewServeMux() mux.HandleFunc("/", app.indexUI) mux.HandleFunc("GET /health", app.health) mux.HandleFunc("POST /api/auth/login", app.handleLogin) mux.HandleFunc("POST /api/auth/register", app.handleRegister) mux.HandleFunc("POST /api/auth/logout", app.handleLogout) mux.HandleFunc("GET /api/auth/me", app.handleMe) mux.Handle("GET /api/admin/users", app.auth(http.HandlerFunc(app.handleAdminListUsers))) mux.Handle("POST /api/admin/users", app.auth(http.HandlerFunc(app.handleAdminSaveUser))) mux.Handle("DELETE /api/admin/users", app.auth(http.HandlerFunc(app.handleAdminDeleteUser))) mux.Handle("POST /api/upload", app.auth(http.HandlerFunc(app.upload))) mux.Handle("POST /api/upload/batch", app.auth(http.HandlerFunc(app.uploadBatch))) mux.Handle("POST /api/upload/init", app.auth(http.HandlerFunc(app.uploadChunkInit))) mux.Handle("POST /api/upload/chunk", app.auth(http.HandlerFunc(app.uploadChunkPart))) mux.Handle("POST /api/upload/complete", app.auth(http.HandlerFunc(app.uploadChunkComplete))) mux.Handle("GET /api/files", app.auth(http.HandlerFunc(app.listFiles))) mux.Handle("PUT /api/files", app.auth(http.HandlerFunc(app.renameFile))) mux.Handle("DELETE /api/files", app.auth(http.HandlerFunc(app.deleteFile))) mux.Handle("GET /api/download", app.auth(http.HandlerFunc(app.download))) mux.Handle("GET /api/shares", app.auth(http.HandlerFunc(app.handleListShares))) mux.Handle("GET /api/shares/file", app.auth(http.HandlerFunc(app.handleGetFileShare))) mux.Handle("POST /api/shares", app.auth(http.HandlerFunc(app.handleUpsertShare))) mux.Handle("POST /api/shares/approve", app.auth(http.HandlerFunc(app.handleApproveShareRequest))) mux.HandleFunc("GET /api/public/share", app.handlePublicGetShare) mux.HandleFunc("POST /api/public/share/request", app.handlePublicRequestShareAccess) mux.HandleFunc("GET /api/public/share/download", app.handlePublicShareDownload) server := &http.Server{ Addr: cfg.ListenAddr, Handler: securityHeaders(mux), ReadHeaderTimeout: 60 * time.Second, ReadTimeout: 2 * time.Hour, WriteTimeout: 2 * time.Hour, IdleTimeout: 5 * time.Minute, } log.Printf("Hugging Face Bucket API listening on %s (bucket=%s)", cfg.ListenAddr, bucketID) log.Fatal(server.ListenAndServe()) } func loadDotEnv(filename string) { f, err := os.Open(filename) if err != nil { return } defer f.Close() scanner := bufio.NewScanner(f) for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) if line == "" || strings.HasPrefix(line, "#") { continue } k, v, ok := strings.Cut(line, "=") if !ok { continue } k = strings.TrimSpace(k) v = strings.Trim(strings.TrimSpace(v), `"'`) if k != "" && os.Getenv(k) == "" { _ = os.Setenv(k, v) } } } func loadConfig() (Config, error) { loadDotEnv(".env") cfg := Config{ HFCLI: env("HF_CLI", "hf"), HFToken: os.Getenv("HF_TOKEN"), APIKey: os.Getenv("API_KEY"), ListenAddr: env("LISTEN_ADDR", ":8080"), TempDir: env("TEMP_DIR", os.TempDir()), MaxUpload: 10 << 30, CommandTime: 30 * time.Minute, } if cfg.HFToken == "" { return cfg, errors.New("HF_TOKEN is required") } if cfg.APIKey == "" { return cfg, errors.New("API_KEY is required; set a private key for API access") } if n := os.Getenv("MAX_UPLOAD_BYTES"); n != "" { v, err := strconv.ParseInt(n, 10, 64) if err != nil || v <= 0 { return cfg, fmt.Errorf("invalid MAX_UPLOAD_BYTES: %q", n) } cfg.MaxUpload = v } return cfg, nil } func env(k, fallback string) string { if v := os.Getenv(k); v != "" { return v } return fallback } func (a *App) auth(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { u, ok := a.resolveRequestUser(r) if !ok { writeJSON(w, http.StatusUnauthorized, APIError{Error: "unauthorized", Detail: "Vui lòng đăng nhập hoặc cung cấp API key hợp lệ"}) return } ctx := context.WithValue(r.Context(), userContextKey, u) next.ServeHTTP(w, r.WithContext(ctx)) }) } func currentUser(r *http.Request) UserAccount { if u, ok := r.Context().Value(userContextKey).(UserAccount); ok { return u } return UserAccount{} } func (a *App) indexUI(w http.ResponseWriter, r *http.Request) { // Block unknown /api/* routes or scanner requests (/.env, /.git, etc.) from being proxied to Next.js if strings.HasPrefix(r.URL.Path, "/api/") || r.URL.Path == "/api" || strings.Contains(r.URL.Path, "/.") { writeJSON(w, http.StatusNotFound, APIError{Error: "not_found", Detail: "Endpoint không tồn tại"}) return } if feURL := os.Getenv("FRONTEND_URL"); feURL != "" { if target, err := url.Parse(feURL); err == nil { proxy := httputil.NewSingleHostReverseProxy(target) proxy.ErrorHandler = func(rw http.ResponseWriter, req *http.Request, e error) { if req.URL.Path == "/" || req.URL.Path == "/index.html" { if data, rErr := webFS.ReadFile("web/index.html"); rErr == nil { rw.Header().Set("Content-Type", "text/html; charset=utf-8") rw.WriteHeader(http.StatusOK) _, _ = rw.Write(data) return } } http.Error(rw, "Frontend starting up...", http.StatusBadGateway) } proxy.ServeHTTP(w, r) return } } if r.URL.Path != "/" && r.URL.Path != "/index.html" { http.NotFound(w, r) return } data, err := webFS.ReadFile("web/index.html") if err != nil { http.Error(w, "UI not found", http.StatusInternalServerError) return } w.Header().Set("Content-Type", "text/html; charset=utf-8") w.WriteHeader(http.StatusOK) _, _ = w.Write(data) } func securityHeaders(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("X-Content-Type-Options", "nosniff") if strings.HasPrefix(r.URL.Path, "/api/") || r.URL.Path == "/health" { w.Header().Set("Cache-Control", "no-store") } origin := r.Header.Get("Origin") if origin != "" { w.Header().Set("Access-Control-Allow-Origin", origin) w.Header().Set("Access-Control-Allow-Credentials", "true") w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Authorization, X-API-Key") w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS") } if r.Method == http.MethodOptions { w.WriteHeader(http.StatusNoContent) return } next.ServeHTTP(w, r) }) } func (a *App) health(w http.ResponseWriter, r *http.Request) { dbEngine := "local-json" if a.mongo != nil { dbEngine = "mongodb-atlas (" + a.mongo.dbName + ")" } if a.pool != nil { if res, err := a.pool.Call(map[string]any{"op": "ping"}); err == nil { ver, _ := res["version"].(string) writeJSON(w, http.StatusOK, map[string]any{"ok": true, "hf_cli": true, "turbo_worker": true, "database": dbEngine, "version": ver, "bucket": bucketID}) return } } ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second) defer cancel() out, err := exec.CommandContext(ctx, a.cfg.HFCLI, "--version").CombinedOutput() if err != nil { writeJSON(w, http.StatusServiceUnavailable, map[string]any{"ok": false, "hf_cli": false, "detail": strings.TrimSpace(string(out))}) return } writeJSON(w, http.StatusOK, map[string]any{"ok": true, "hf_cli": true, "version": strings.TrimSpace(string(out)), "bucket": bucketID}) } func (a *App) upload(w http.ResponseWriter, r *http.Request) { start := time.Now() u := currentUser(r) if u.Role != "admin" && !u.Permissions.CanUpload { writeJSON(w, http.StatusForbidden, APIError{Error: "forbidden", Detail: "Tài khoản của bạn không được cấp quyền tải tệp lên"}) return } r.Body = http.MaxBytesReader(w, r.Body, a.cfg.MaxUpload+(1<<20)) mr, err := r.MultipartReader() if err != nil { writeJSON(w, http.StatusBadRequest, APIError{Error: "invalid_multipart", Detail: err.Error()}) return } var remotePath string var tmpName string var fileName string var size int64 copyBuf := make([]byte, 4<<20) // 4 MiB streaming buffer defer func() { if tmpName != "" { _ = os.Remove(tmpName) } }() for { part, err := mr.NextPart() if errors.Is(err, io.EOF) { break } if err != nil { writeJSON(w, http.StatusBadRequest, APIError{Error: "multipart_stream_error", Detail: err.Error()}) return } formName := part.FormName() if formName == "path" { b, _ := io.ReadAll(io.LimitReader(part, 4096)) remotePath = strings.TrimSpace(string(b)) _ = part.Close() } else if formName == "file" { fileName = part.FileName() tmp, err := os.CreateTemp(a.cfg.TempDir, "hf-upload-*") if err != nil { _ = part.Close() writeJSON(w, http.StatusInternalServerError, APIError{Error: "temp_file_error", Detail: err.Error()}) return } tmpName = tmp.Name() n, copyErr := io.CopyBuffer(tmp, io.LimitReader(part, a.cfg.MaxUpload+1), copyBuf) closeErr := tmp.Close() _ = part.Close() if copyErr != nil { writeJSON(w, http.StatusBadRequest, APIError{Error: "upload_read_error", Detail: copyErr.Error()}) return } if closeErr != nil { writeJSON(w, http.StatusInternalServerError, APIError{Error: "temp_file_error", Detail: closeErr.Error()}) return } size = n if size > a.cfg.MaxUpload { writeJSON(w, http.StatusRequestEntityTooLarge, APIError{Error: "file_too_large"}) return } } else { _ = part.Close() } } if tmpName == "" { writeJSON(w, http.StatusBadRequest, APIError{Error: "missing_file", Detail: "send multipart field named 'file'"}) return } if remotePath == "" { remotePath = fileName } remotePath, err = cleanRemotePath(remotePath) if err != nil { writeJSON(w, http.StatusBadRequest, APIError{Error: "invalid_path", Detail: err.Error()}) return } remotePath = a.resolveUniqueRemotePath(scopeUserRemotePath(u.Username, remotePath)) a.evictDownloadCache(remotePath) // Fast path: Persistent Python Xet Worker Pool (no process startup overhead) if a.pool != nil { _, err = a.pool.Call(map[string]any{ "op": "upload", "bucket": bucketID, "files": []map[string]string{ {"local": tmpName, "remote": remotePath}, }, }) if err == nil { if a.owners != nil { a.owners.SetOwner(remotePath, u.Username) } if a.mongo != nil { a.mongo.UpsertFileMeta(remotePath, u.Username, size) } a.invalidateListCache() writeJSON(w, http.StatusOK, map[string]any{ "ok": true, "bucket": bucketID, "path": remotePath, "owner": u.Username, "size": size, "duration_ms": time.Since(start).Milliseconds(), "engine": "xet-worker-pool", "message": "upload completed", }) return } } // Fallback: CLI dest := "hf://buckets/" + bucketID + "/" + remotePath out, err := a.runHF(r.Context(), "buckets", "cp", tmpName, dest) if err != nil { writeJSON(w, http.StatusBadGateway, APIError{Error: "huggingface_upload_failed", Detail: out}) return } if a.owners != nil { a.owners.SetOwner(remotePath, u.Username) } a.invalidateListCache() writeJSON(w, http.StatusOK, map[string]any{ "ok": true, "bucket": bucketID, "path": remotePath, "owner": u.Username, "size": size, "duration_ms": time.Since(start).Milliseconds(), "message": "upload completed", }) } func (a *App) uploadBatch(w http.ResponseWriter, r *http.Request) { start := time.Now() u := currentUser(r) if u.Role != "admin" && !u.Permissions.CanUpload { writeJSON(w, http.StatusForbidden, APIError{Error: "forbidden", Detail: "Tài khoản của bạn không được cấp quyền tải tệp lên"}) return } r.Body = http.MaxBytesReader(w, r.Body, a.cfg.MaxUpload+(1<<20)) mr, err := r.MultipartReader() if err != nil { writeJSON(w, http.StatusBadRequest, APIError{Error: "invalid_multipart", Detail: err.Error()}) return } var folderPrefix string type stagedFile struct { local string remote string size int64 } var staged []stagedFile copyBuf := make([]byte, 4<<20) defer func() { for _, sf := range staged { _ = os.Remove(sf.local) } }() for { part, err := mr.NextPart() if errors.Is(err, io.EOF) { break } if err != nil { writeJSON(w, http.StatusBadRequest, APIError{Error: "multipart_stream_error", Detail: err.Error()}) return } formName := part.FormName() if formName == "folder" { b, _ := io.ReadAll(io.LimitReader(part, 4096)) folderPrefix = strings.Trim(strings.TrimSpace(string(b)), "/") _ = part.Close() } else if formName == "files" || formName == "file" { fn := part.FileName() if fn == "" { _ = part.Close() continue } remote := fn if folderPrefix != "" { remote = folderPrefix + "/" + fn } clean, cleanErr := cleanRemotePath(remote) if cleanErr != nil { _ = part.Close() writeJSON(w, http.StatusBadRequest, APIError{Error: "invalid_path", Detail: cleanErr.Error()}) return } clean = a.resolveUniqueRemotePath(scopeUserRemotePath(u.Username, clean)) a.evictDownloadCache(clean) tmp, err := os.CreateTemp(a.cfg.TempDir, "hf-batch-*") if err != nil { _ = part.Close() writeJSON(w, http.StatusInternalServerError, APIError{Error: "temp_file_error", Detail: err.Error()}) return } n, copyErr := io.CopyBuffer(tmp, io.LimitReader(part, a.cfg.MaxUpload+1), copyBuf) _ = tmp.Close() _ = part.Close() if copyErr != nil { writeJSON(w, http.StatusBadRequest, APIError{Error: "upload_read_error", Detail: copyErr.Error()}) return } staged = append(staged, stagedFile{local: tmp.Name(), remote: clean, size: n}) } else { _ = part.Close() } } if len(staged) == 0 { writeJSON(w, http.StatusBadRequest, APIError{Error: "missing_files", Detail: "no files provided in batch"}) return } pairs := make([]map[string]string, 0, len(staged)) var totalBytes int64 uploadedPaths := make([]string, 0, len(staged)) for _, sf := range staged { pairs = append(pairs, map[string]string{"local": sf.local, "remote": sf.remote}) totalBytes += sf.size uploadedPaths = append(uploadedPaths, sf.remote) } if a.pool != nil { if _, err := a.pool.Call(map[string]any{"op": "upload", "bucket": bucketID, "files": pairs}); err != nil { writeJSON(w, http.StatusBadGateway, APIError{Error: "batch_upload_failed", Detail: err.Error()}) return } } else { for _, sf := range staged { dest := "hf://buckets/" + bucketID + "/" + sf.remote if out, err := a.runHF(r.Context(), "buckets", "cp", sf.local, dest); err != nil { writeJSON(w, http.StatusBadGateway, APIError{Error: "huggingface_upload_failed", Detail: out}) return } } } if a.owners != nil { a.owners.SetOwnersBatch(uploadedPaths, u.Username) } if a.mongo != nil { for _, sf := range staged { a.mongo.UpsertFileMeta(sf.remote, u.Username, sf.size) } } a.invalidateListCache() writeJSON(w, http.StatusOK, map[string]any{ "ok": true, "bucket": bucketID, "count": len(staged), "paths": uploadedPaths, "owner": u.Username, "total_bytes": totalBytes, "duration_ms": time.Since(start).Milliseconds(), "engine": "xet-batch-commit", }) } func (a *App) cleanupStaleChunkSessions() { now := time.Now() a.chunkSessions.Range(func(key, value any) bool { sess, ok := value.(*chunkUploadSession) if !ok { return true } if now.Sub(sess.createdAt) > 2*time.Hour { sess.mu.Lock() if !sess.closed { sess.closed = true _ = sess.file.Close() _ = os.Remove(sess.tmpPath) a.releaseReservedPath(sess.remotePath) } sess.mu.Unlock() a.chunkSessions.Delete(key) } return true }) } func (a *App) uploadChunkInit(w http.ResponseWriter, r *http.Request) { u := currentUser(r) if u.Role != "admin" && !u.Permissions.CanUpload { writeJSON(w, http.StatusForbidden, APIError{Error: "forbidden", Detail: "Tài khoản của bạn không được cấp quyền tải tệp lên"}) return } a.cleanupStaleChunkSessions() var req struct { Path string `json:"path"` Size int64 `json:"size"` } if err := json.NewDecoder(io.LimitReader(r.Body, 16384)).Decode(&req); err != nil { writeJSON(w, http.StatusBadRequest, APIError{Error: "invalid_json", Detail: err.Error()}) return } if req.Size < 0 || req.Size > a.cfg.MaxUpload { writeJSON(w, http.StatusRequestEntityTooLarge, APIError{Error: "file_too_large"}) return } clean, err := cleanRemotePath(req.Path) if err != nil { writeJSON(w, http.StatusBadRequest, APIError{Error: "invalid_path", Detail: err.Error()}) return } remotePath := a.resolveUniqueRemotePath(scopeUserRemotePath(u.Username, clean)) a.evictDownloadCache(remotePath) tmp, err := os.CreateTemp(a.cfg.TempDir, "hf-chunked-*") if err != nil { a.releaseReservedPath(remotePath) writeJSON(w, http.StatusInternalServerError, APIError{Error: "temp_file_error", Detail: err.Error()}) return } if req.Size > 0 { if err := tmp.Truncate(req.Size); err != nil { _ = tmp.Close() _ = os.Remove(tmp.Name()) a.releaseReservedPath(remotePath) writeJSON(w, http.StatusInternalServerError, APIError{Error: "temp_alloc_error", Detail: err.Error()}) return } } uploadID := randomHex(16) sess := &chunkUploadSession{ id: uploadID, owner: u.Username, remotePath: remotePath, tmpPath: tmp.Name(), file: tmp, totalSize: req.Size, createdAt: time.Now(), } a.chunkSessions.Store(uploadID, sess) writeJSON(w, http.StatusOK, map[string]any{ "ok": true, "upload_id": uploadID, "path": remotePath, "chunk_size": 4 << 20, // 4 MiB default chunk }) } func (a *App) uploadChunkPart(w http.ResponseWriter, r *http.Request) { u := currentUser(r) uploadID := strings.TrimSpace(r.URL.Query().Get("id")) offsetStr := strings.TrimSpace(r.URL.Query().Get("offset")) offset, err := strconv.ParseInt(offsetStr, 10, 64) if uploadID == "" || err != nil || offset < 0 { writeJSON(w, http.StatusBadRequest, APIError{Error: "invalid_params", Detail: "missing id or valid offset"}) return } val, ok := a.chunkSessions.Load(uploadID) if !ok { writeJSON(w, http.StatusNotFound, APIError{Error: "session_not_found", Detail: "Phiên tải lên không tồn tại hoặc đã hết hạn"}) return } sess := val.(*chunkUploadSession) if sess.owner != u.Username && u.Role != "admin" { writeJSON(w, http.StatusForbidden, APIError{Error: "forbidden"}) return } buf := make([]byte, 1<<20) // 1 MiB copy buffer per parallel stream ow := &offsetWriter{f: sess.file, off: offset} written, copyErr := io.CopyBuffer(ow, io.LimitReader(r.Body, 64<<20), buf) if copyErr != nil { writeJSON(w, http.StatusBadRequest, APIError{Error: "chunk_write_error", Detail: copyErr.Error()}) return } writeJSON(w, http.StatusOK, map[string]any{ "ok": true, "written": written, "offset": offset, }) } func (a *App) uploadChunkComplete(w http.ResponseWriter, r *http.Request) { start := time.Now() u := currentUser(r) uploadID := strings.TrimSpace(r.URL.Query().Get("id")) if uploadID == "" { writeJSON(w, http.StatusBadRequest, APIError{Error: "missing_id"}) return } val, ok := a.chunkSessions.LoadAndDelete(uploadID) if !ok { writeJSON(w, http.StatusNotFound, APIError{Error: "session_not_found"}) return } sess := val.(*chunkUploadSession) if sess.owner != u.Username && u.Role != "admin" { writeJSON(w, http.StatusForbidden, APIError{Error: "forbidden"}) return } sess.mu.Lock() sess.closed = true _ = sess.file.Close() sess.mu.Unlock() defer func() { _ = os.Remove(sess.tmpPath) }() if a.pool != nil { _, err := a.pool.Call(map[string]any{ "op": "upload", "bucket": bucketID, "files": []map[string]string{ {"local": sess.tmpPath, "remote": sess.remotePath}, }, }) if err != nil { a.releaseReservedPath(sess.remotePath) writeJSON(w, http.StatusBadGateway, APIError{Error: "huggingface_upload_failed", Detail: err.Error()}) return } } else { dest := "hf://buckets/" + bucketID + "/" + sess.remotePath if out, err := a.runHF(r.Context(), "buckets", "cp", sess.tmpPath, dest); err != nil { a.releaseReservedPath(sess.remotePath) writeJSON(w, http.StatusBadGateway, APIError{Error: "huggingface_upload_failed", Detail: out}) return } } if a.owners != nil { a.owners.SetOwner(sess.remotePath, sess.owner) } if a.mongo != nil { a.mongo.UpsertFileMeta(sess.remotePath, sess.owner, sess.totalSize) } a.invalidateListCache() writeJSON(w, http.StatusOK, map[string]any{ "ok": true, "bucket": bucketID, "path": sess.remotePath, "owner": sess.owner, "size": sess.totalSize, "duration_ms": time.Since(start).Milliseconds(), "engine": "xet-parallel-chunked", }) } func parseBucketLsLine(line string) (FileItem, bool) { trimmed := strings.TrimSpace(line) if trimmed == "" { return FileItem{}, false } fields := strings.Fields(trimmed) if len(fields) >= 4 { if size, err := strconv.ParseInt(fields[0], 10, 64); err == nil { datePart := fields[1] timePart := fields[2] if len(datePart) == 10 && strings.Count(datePart, "-") == 2 && strings.Count(timePart, ":") == 2 { idx := strings.Index(trimmed, timePart) if idx != -1 { filePath := strings.TrimSpace(trimmed[idx+len(timePart):]) return FileItem{ Path: filePath, Size: size, UpdatedAt: datePart + " " + timePart, }, true } } } } return FileItem{Path: trimmed}, true } func (a *App) filterUserFiles(u UserAccount, all []FileItem) ([]string, []FileItem) { lines := make([]string, 0, len(all)) items := make([]FileItem, 0, len(all)) for _, it := range all { if a.owners != nil && !a.owners.CanAccess(u, it.Path) { continue } owner := "admin" if a.owners != nil { owner = a.owners.GetOwnerForUser(u, it.Path) } it.Owner = owner lines = append(lines, it.Path) items = append(items, it) } return lines, items } func (a *App) listFiles(w http.ResponseWriter, r *http.Request) { u := currentUser(r) prefix := strings.TrimSpace(r.URL.Query().Get("prefix")) if prefix != "" { clean, err := cleanRemotePath(prefix) if err != nil { writeJSON(w, http.StatusBadRequest, APIError{Error: "invalid_prefix", Detail: err.Error()}) return } prefix = clean } if prefix == "" { a.listCacheMu.RLock() cached := a.listCache cachedAt := a.listCacheTime a.listCacheMu.RUnlock() if !cachedAt.IsZero() && time.Since(cachedAt) < 5*time.Second { lines, items := a.filterUserFiles(u, cached) writeJSON(w, http.StatusOK, map[string]any{ "ok": true, "bucket": bucketID, "prefix": prefix, "count": len(lines), "entries": lines, "items": items, "engine": "xet-memory-cache", }) return } } if a.pool != nil { if res, err := a.pool.Call(map[string]any{"op": "ls", "bucket": bucketID, "prefix": prefix}); err == nil { rawItems, _ := res["items"].([]any) allItems := make([]FileItem, 0, len(rawItems)) for _, ri := range rawItems { if m, ok := ri.(map[string]any); ok { p, _ := m["path"].(string) sz, _ := m["size"].(float64) up, _ := m["updated_at"].(string) allItems = append(allItems, FileItem{Path: p, Size: int64(sz), UpdatedAt: up}) } } if prefix == "" { a.listCacheMu.Lock() a.listCache = allItems a.listCacheTime = time.Now() a.listCacheMu.Unlock() } lines, items := a.filterUserFiles(u, allItems) writeJSON(w, http.StatusOK, map[string]any{ "ok": true, "bucket": bucketID, "prefix": prefix, "count": len(lines), "entries": lines, "items": items, "engine": "xet-worker-pool", }) return } } args := []string{"buckets", "ls", bucketID} if prefix != "" { args[2] += "/" + prefix } args = append(args, "-R") out, err := a.runHF(r.Context(), args...) if err != nil { writeJSON(w, http.StatusBadGateway, APIError{Error: "huggingface_list_failed", Detail: out}) return } lines := make([]string, 0) items := make([]FileItem, 0) for _, line := range strings.Split(strings.ReplaceAll(out, "\r\n", "\n"), "\n") { if item, ok := parseBucketLsLine(line); ok { if a.owners != nil && !a.owners.CanAccess(u, item.Path) { continue } if a.owners != nil { item.Owner = a.owners.GetOwner(item.Path) } lines = append(lines, item.Path) items = append(items, item) } } writeJSON(w, http.StatusOK, map[string]any{ "ok": true, "bucket": bucketID, "prefix": prefix, "count": len(lines), "entries": lines, "items": items, "raw": out, }) } func (a *App) deleteFile(w http.ResponseWriter, r *http.Request) { u := currentUser(r) if u.Role != "admin" && !u.Permissions.CanDelete { writeJSON(w, http.StatusForbidden, APIError{Error: "forbidden", Detail: "Chỉ Admin hoặc tài khoản có quyền Xóa mới thực hiện được thao tác này"}) return } remote, err := cleanRemotePath(r.URL.Query().Get("path")) if err != nil { writeJSON(w, http.StatusBadRequest, APIError{Error: "invalid_path", Detail: err.Error()}) return } if a.owners != nil && !a.owners.CanAccess(u, remote) { writeJSON(w, http.StatusForbidden, APIError{Error: "forbidden", Detail: "Bạn chỉ có thể xóa tệp tin do chính mình tải lên"}) return } if a.pool != nil { if _, err := a.pool.Call(map[string]any{"op": "rm", "bucket": bucketID, "remote": remote}); err == nil { if a.owners != nil { a.owners.Delete(remote) } if a.shares != nil { a.shares.DeleteByPath(remote) } a.releaseReservedPath(remote) a.evictDownloadCache(remote) a.invalidateListCache() writeJSON(w, http.StatusOK, map[string]any{"ok": true, "bucket": bucketID, "path": remote, "message": "file deleted"}) return } } target := bucketID + "/" + remote out, err := a.runHF(r.Context(), "buckets", "rm", target, "--yes") if err != nil { writeJSON(w, http.StatusBadGateway, APIError{Error: "huggingface_delete_failed", Detail: out}) return } if a.owners != nil { a.owners.Delete(remote) } if a.shares != nil { a.shares.DeleteByPath(remote) } a.releaseReservedPath(remote) a.evictDownloadCache(remote) a.invalidateListCache() writeJSON(w, http.StatusOK, map[string]any{"ok": true, "bucket": bucketID, "path": remote, "message": "file deleted"}) } func (a *App) renameFile(w http.ResponseWriter, r *http.Request) { u := currentUser(r) if u.Role != "admin" && !u.Permissions.CanRename { writeJSON(w, http.StatusForbidden, APIError{Error: "forbidden", Detail: "Chỉ Admin hoặc tài khoản có quyền Chỉnh sửa mới được đổi tên/di chuyển tệp"}) return } var body struct { OldPath string `json:"old_path"` NewPath string `json:"new_path"` } if err := json.NewDecoder(r.Body).Decode(&body); err != nil { writeJSON(w, http.StatusBadRequest, APIError{Error: "invalid_json", Detail: err.Error()}) return } oldClean, err := cleanRemotePath(body.OldPath) if err != nil { writeJSON(w, http.StatusBadRequest, APIError{Error: "invalid_old_path", Detail: err.Error()}) return } if a.owners != nil && !a.owners.CanAccess(u, oldClean) { writeJSON(w, http.StatusForbidden, APIError{Error: "forbidden", Detail: "Bạn chỉ có thể đổi tên tệp tin do chính mình tải lên"}) return } newClean, err := cleanRemotePath(body.NewPath) if err != nil { writeJSON(w, http.StatusBadRequest, APIError{Error: "invalid_new_path", Detail: err.Error()}) return } if parts := strings.SplitN(oldClean, "/", 2); len(parts) == 2 && a.users != nil { newClean = scopeUserRemotePath(parts[0], newClean) } if oldClean == newClean { writeJSON(w, http.StatusOK, map[string]any{"ok": true, "old_path": oldClean, "new_path": newClean, "message": "unchanged"}) return } // Zero-data-transfer server-side Xet hash rename via persistent worker pool if a.pool != nil { if _, err := a.pool.Call(map[string]any{ "op": "mv", "bucket": bucketID, "old_path": oldClean, "new_path": newClean, }); err == nil { if a.owners != nil { a.owners.Rename(oldClean, newClean) } if a.shares != nil { a.shares.RenamePath(oldClean, newClean) } a.releaseReservedPath(oldClean) a.evictDownloadCache(oldClean) a.evictDownloadCache(newClean) a.invalidateListCache() writeJSON(w, http.StatusOK, map[string]any{ "ok": true, "bucket": bucketID, "old_path": oldClean, "new_path": newClean, "engine": "xet-zero-copy-hash", "message": "file renamed instantaneously via Xet hash", }) return } } tmpDir, err := os.MkdirTemp(a.cfg.TempDir, "hf-rename-*") if err != nil { writeJSON(w, http.StatusInternalServerError, APIError{Error: "temp_dir_error", Detail: err.Error()}) return } defer os.RemoveAll(tmpDir) local := filepath.Join(tmpDir, filepath.Base(oldClean)) src := "hf://buckets/" + bucketID + "/" + oldClean dest := "hf://buckets/" + bucketID + "/" + newClean if out, err := a.runHF(r.Context(), "buckets", "cp", src, local); err != nil { writeJSON(w, http.StatusBadGateway, APIError{Error: "rename_download_failed", Detail: out}) return } if out, err := a.runHF(r.Context(), "buckets", "cp", local, dest); err != nil { writeJSON(w, http.StatusBadGateway, APIError{Error: "rename_upload_failed", Detail: out}) return } if out, err := a.runHF(r.Context(), "buckets", "rm", bucketID+"/"+oldClean, "--yes"); err != nil { writeJSON(w, http.StatusBadGateway, APIError{Error: "rename_cleanup_failed", Detail: out}) return } if a.owners != nil { a.owners.Rename(oldClean, newClean) } if a.shares != nil { a.shares.RenamePath(oldClean, newClean) } writeJSON(w, http.StatusOK, map[string]any{ "ok": true, "bucket": bucketID, "old_path": oldClean, "new_path": newClean, "message": "file renamed successfully", }) } func (a *App) download(w http.ResponseWriter, r *http.Request) { u := currentUser(r) remote, err := cleanRemotePath(r.URL.Query().Get("path")) if err != nil { writeJSON(w, http.StatusBadRequest, APIError{Error: "invalid_path", Detail: err.Error()}) return } if a.owners != nil && !a.owners.CanAccess(u, remote) { writeJSON(w, http.StatusForbidden, APIError{Error: "forbidden", Detail: "Bạn không có quyền truy cập tệp tin của người dùng khác"}) return } a.serveRemoteFile(w, r, remote) } func (a *App) serveRemoteFile(w http.ResponseWriter, r *http.Request, remote string) { cacheDir := filepath.Join(a.cfg.TempDir, "hf-cache") if err := os.MkdirAll(cacheDir, 0700); err != nil { writeJSON(w, http.StatusInternalServerError, APIError{Error: "cache_dir_error", Detail: err.Error()}) return } hash := sha256.Sum256([]byte(remote)) ext := filepath.Ext(remote) cachedFile := filepath.Join(cacheDir, hex.EncodeToString(hash[:16])+ext) mu := a.fileLock(cachedFile) mu.Lock() needFetch := true if st, err := os.Stat(cachedFile); err == nil && st.Size() > 0 && time.Since(st.ModTime()) < 30*time.Minute { needFetch = false } if needFetch { partFile := cachedFile + ".part." + strconv.FormatInt(time.Now().UnixNano(), 36) fetched := false if a.pool != nil { if _, err := a.pool.Call(map[string]any{ "op": "download", "bucket": bucketID, "remote": remote, "local": partFile, }); err == nil { fetched = true } } if !fetched { src := "hf://buckets/" + bucketID + "/" + remote out, err := a.runHF(context.Background(), "buckets", "cp", src, partFile) if err != nil { _ = os.Remove(partFile) mu.Unlock() writeJSON(w, http.StatusBadGateway, APIError{Error: "huggingface_download_failed", Detail: out}) return } } _ = os.Remove(cachedFile) if err := os.Rename(partFile, cachedFile); err != nil { _ = os.Remove(partFile) mu.Unlock() writeJSON(w, http.StatusInternalServerError, APIError{Error: "cache_rename_failed", Detail: err.Error()}) return } } mu.Unlock() f, err := os.Open(cachedFile) if err != nil { writeJSON(w, http.StatusBadGateway, APIError{Error: "download_missing", Detail: "CLI completed but output file was not found"}) return } defer f.Close() st, err := f.Stat() if err != nil { writeJSON(w, http.StatusInternalServerError, APIError{Error: "file_stat_failed", Detail: err.Error()}) return } name := filepath.Base(remote) dispType := "attachment" if r.URL.Query().Get("inline") == "1" { dispType = "inline" } w.Header().Set("Content-Disposition", mime.FormatMediaType(dispType, map[string]string{"filename": name})) ct := mime.TypeByExtension(filepath.Ext(name)) if ct != "" { w.Header().Set("Content-Type", ct) } http.ServeContent(w, r, name, st.ModTime(), f) } // --- Quick Share Handlers --- func (a *App) handleListShares(w http.ResponseWriter, r *http.Request) { u := currentUser(r) if a.shares == nil { writeJSON(w, http.StatusOK, map[string]any{"ok": true, "shares": []any{}}) return } list := a.shares.ListForUser(u) writeJSON(w, http.StatusOK, map[string]any{"ok": true, "shares": list}) } func (a *App) handleGetFileShare(w http.ResponseWriter, r *http.Request) { u := currentUser(r) remote, err := cleanRemotePath(r.URL.Query().Get("path")) if err != nil { writeJSON(w, http.StatusBadRequest, APIError{Error: "invalid_path", Detail: err.Error()}) return } if a.owners != nil && !a.owners.CanAccess(u, remote) { writeJSON(w, http.StatusForbidden, APIError{Error: "forbidden", Detail: "Bạn không có quyền quản lý chia sẻ tệp này"}) return } if a.shares == nil { writeJSON(w, http.StatusOK, map[string]any{"ok": true, "share": nil}) return } sh, ok := a.shares.GetByPath(remote) if !ok { writeJSON(w, http.StatusOK, map[string]any{"ok": true, "share": nil}) return } writeJSON(w, http.StatusOK, map[string]any{"ok": true, "share": sh}) } func (a *App) handleUpsertShare(w http.ResponseWriter, r *http.Request) { u := currentUser(r) var body struct { Path string `json:"path"` Size int64 `json:"size"` Enabled bool `json:"enabled"` RequireApproval bool `json:"require_approval"` Regenerate bool `json:"regenerate"` } if err := json.NewDecoder(r.Body).Decode(&body); err != nil { writeJSON(w, http.StatusBadRequest, APIError{Error: "invalid_json", Detail: err.Error()}) return } remote, err := cleanRemotePath(body.Path) if err != nil { writeJSON(w, http.StatusBadRequest, APIError{Error: "invalid_path", Detail: err.Error()}) return } if a.owners != nil && !a.owners.CanAccess(u, remote) { writeJSON(w, http.StatusForbidden, APIError{Error: "forbidden", Detail: "Bạn chỉ có thể chia sẻ tệp thuộc quyền sở hữu của mình"}) return } owner := u.Username if a.owners != nil { owner = a.owners.GetOwnerForUser(u, remote) } sh := a.shares.UpsertShare(remote, owner, body.Size, body.Enabled, body.RequireApproval, body.Regenerate) writeJSON(w, http.StatusOK, map[string]any{"ok": true, "share": sh}) } func (a *App) handleApproveShareRequest(w http.ResponseWriter, r *http.Request) { u := currentUser(r) var body struct { ShareID string `json:"share_id"` RequestID string `json:"request_id"` Status string `json:"status"` // "approved" or "rejected" } if err := json.NewDecoder(r.Body).Decode(&body); err != nil { writeJSON(w, http.StatusBadRequest, APIError{Error: "invalid_json", Detail: err.Error()}) return } if body.Status != "approved" && body.Status != "rejected" { writeJSON(w, http.StatusBadRequest, APIError{Error: "invalid_status", Detail: "status phải là approved hoặc rejected"}) return } sh, ok := a.shares.GetByID(body.ShareID) if !ok { writeJSON(w, http.StatusNotFound, APIError{Error: "not_found", Detail: "Không tìm thấy liên kết chia sẻ"}) return } if u.Role != "admin" && !strings.EqualFold(sh.Owner, u.Username) && (a.owners != nil && !a.owners.CanAccess(u, sh.Path)) { writeJSON(w, http.StatusForbidden, APIError{Error: "forbidden", Detail: "Bạn không có quyền duyệt yêu cầu của tệp này"}) return } updated, ok := a.shares.UpdateRequestStatus(body.ShareID, body.RequestID, body.Status) if !ok { writeJSON(w, http.StatusNotFound, APIError{Error: "request_not_found", Detail: "Không tìm thấy yêu cầu truy cập"}) return } writeJSON(w, http.StatusOK, map[string]any{"ok": true, "share": updated}) } func (a *App) handlePublicGetShare(w http.ResponseWriter, r *http.Request) { shareID := strings.TrimSpace(r.URL.Query().Get("id")) visitorToken := strings.TrimSpace(r.URL.Query().Get("visitor_token")) if shareID == "" || a.shares == nil { writeJSON(w, http.StatusNotFound, APIError{Error: "not_found", Detail: "Liên kết chia sẻ không hợp lệ"}) return } sh, granted, reqStatus, visitorName := a.shares.CheckVisitorAccess(shareID, visitorToken) if sh == nil { writeJSON(w, http.StatusForbidden, APIError{Error: "share_disabled", Detail: "Liên kết chia sẻ này hiện đang tắt hoặc không tồn tại"}) return } // Also grant immediately if the viewer is logged in as the file owner or admin if !granted { if u, ok := a.resolveRequestUser(r); ok { if u.Role == "admin" || strings.EqualFold(u.Username, sh.Owner) || (a.owners != nil && a.owners.CanAccess(u, sh.Path)) { granted = true reqStatus = "approved" } } } writeJSON(w, http.StatusOK, map[string]any{ "ok": true, "access_granted": granted, "request_status": reqStatus, "visitor_name": visitorName, "share": map[string]any{ "id": sh.ID, "file_name": sh.FileName, "size": sh.Size, "owner": sh.Owner, "enabled": sh.Enabled, "require_approval": sh.RequireApproval, "created_at": sh.CreatedAt, "updated_at": sh.UpdatedAt, }, }) } func (a *App) handlePublicRequestShareAccess(w http.ResponseWriter, r *http.Request) { var body struct { ShareID string `json:"share_id"` VisitorToken string `json:"visitor_token"` VisitorName string `json:"visitor_name"` Note string `json:"note"` } if err := json.NewDecoder(r.Body).Decode(&body); err != nil { writeJSON(w, http.StatusBadRequest, APIError{Error: "invalid_json", Detail: err.Error()}) return } sh, req, ok := a.shares.SubmitAccessRequest(body.ShareID, body.VisitorToken, body.VisitorName, body.Note) if !ok || sh == nil || req == nil { writeJSON(w, http.StatusForbidden, APIError{Error: "share_disabled", Detail: "Liên kết chia sẻ đã bị tắt hoặc không tồn tại"}) return } writeJSON(w, http.StatusOK, map[string]any{ "ok": true, "request_id": req.RequestID, "visitor_token": req.VisitorToken, "request_status": req.Status, }) } func (a *App) handlePublicShareDownload(w http.ResponseWriter, r *http.Request) { shareID := strings.TrimSpace(r.URL.Query().Get("id")) visitorToken := strings.TrimSpace(r.URL.Query().Get("visitor_token")) if shareID == "" || a.shares == nil { writeJSON(w, http.StatusNotFound, APIError{Error: "not_found", Detail: "Liên kết chia sẻ không hợp lệ"}) return } sh, granted, _, _ := a.shares.CheckVisitorAccess(shareID, visitorToken) if sh == nil { writeJSON(w, http.StatusForbidden, APIError{Error: "share_disabled", Detail: "Liên kết chia sẻ này đã bị tắt hoặc không tồn tại"}) return } if !granted { if u, ok := a.resolveRequestUser(r); ok { if u.Role == "admin" || strings.EqualFold(u.Username, sh.Owner) || (a.owners != nil && a.owners.CanAccess(u, sh.Path)) { granted = true } } } if !granted { writeJSON(w, http.StatusForbidden, APIError{Error: "approval_required", Detail: "Bạn cần được chủ sở hữu tệp cho phép trước khi xem hoặc tải xuống"}) return } a.serveRemoteFile(w, r, sh.Path) } func (a *App) runHF(parent context.Context, args ...string) (string, error) { ctx, cancel := context.WithTimeout(parent, a.cfg.CommandTime) defer cancel() cmd := exec.CommandContext(ctx, a.cfg.HFCLI, args...) cmd.Env = append(os.Environ(), "HF_TOKEN="+a.cfg.HFToken, "HF_HUB_DISABLE_PROGRESS_BARS=1") out, err := cmd.CombinedOutput() text := strings.TrimSpace(string(out)) if ctx.Err() != nil { return text, fmt.Errorf("Hugging Face command timed out: %w", ctx.Err()) } if err != nil { return text, fmt.Errorf("hf command failed: %w", err) } return text, nil } func cleanRemotePath(s string) (string, error) { s = strings.TrimSpace(strings.ReplaceAll(s, "\\", "/")) if s == "" { return "", errors.New("path is required") } if strings.HasPrefix(s, "/") || strings.ContainsRune(s, 0) { return "", errors.New("absolute paths and NUL bytes are not allowed") } clean := path.Clean(s) if clean == "." || clean == ".." || strings.HasPrefix(clean, "../") { return "", errors.New("path traversal is not allowed") } return clean, nil } func writeJSON(w http.ResponseWriter, status int, v any) { w.Header().Set("Content-Type", "application/json; charset=utf-8") w.WriteHeader(status) _ = json.NewEncoder(w).Encode(v) }