Spaces:
Running
☁️ Integrate Hugging Face Hub Cloud Storage
Browse filesNEW FEATURE: AI can now write code directly to HF Hub!
MAJOR ADDITIONS:
- code/hub_storage.py: Complete cloud storage module
- HubStorage class with full CRUD operations
- Support for local, hub, and hybrid modes
- Automatic sync between local and cloud
- Dataset repo support for persistent Spaces storage
- Batch operations for efficiency
- File caching and metadata tracking
- Updated code/tools/fs.py: Transparent Hub integration
- read_file: Falls back to Hub if not local
- write_file: Auto-syncs to Hub after local write
- edit_file/multi_edit: Syncs changes back
- list_dir/glob/grep: Merges local + Hub files
- New functions: sync_to_hub, sync_from_hub, get_hub_status
- Updated code/server/routes.py: 11 new API endpoints
- /api/hub_status: Check Hub configuration status
- /api/hub_init: Initialize/configure Hub storage
- /api/hub_sync_push: Push files to Hub
- /api/hub_sync_pull: Pull files from Hub
- /api/hub_full_sync: Bidirectional sync
- /api/hub_list_files: List Hub files
- /api/hub_read_file: Read from Hub only
- /api/hub_write_file: Write to Hub directly (AI uses this!)
- /api/hub_delete_file: Delete from Hub
- /api/hub_create_repo: Create new repo on Hub
- /api/hub_stats: Storage statistics
- Updated app.py: Auto-initialization
- Detects HF Spaces environment automatically
- Uses SPACE_ID and HF_TOKEN env vars
- Creates dataset repo for workspace persistence
- Graceful shutdown syncs to Hub
- Logs storage mode on startup
HOW IT WORKS:
1. On HF Spaces: Auto-detects environment
2. Creates/uses dataset repo for persistence
3. All file ops transparently sync to Hub
4. AI can call hub_write_file() to persist code
5. Workspace survives Space restarts!
CONFIGURATION:
- Set HF_TOKEN env variable (auto-detected)
- Optional: HUB_DATASET_REPO for custom storage
- Optional: HUB_STORAGE_MODE (local/hub/hybrid/auto)
- app.py +114 -1
- code/hub_storage.py +933 -0
- code/server/routes.py +357 -0
- code/tools/fs.py +425 -81
|
@@ -9,6 +9,8 @@ Enhanced version with:
|
|
| 9 |
- Health check endpoint
|
| 10 |
- Session management initialization
|
| 11 |
- Model optimization on startup
|
|
|
|
|
|
|
| 12 |
|
| 13 |
Blog: https://huggingface.co/blog/introducing-gradio-server
|
| 14 |
"""
|
|
@@ -16,6 +18,7 @@ Blog: https://huggingface.co/blog/introducing-gradio-server
|
|
| 16 |
from __future__ import annotations
|
| 17 |
|
| 18 |
import logging
|
|
|
|
| 19 |
import signal
|
| 20 |
import sys
|
| 21 |
import time
|
|
@@ -38,6 +41,17 @@ def setup_signal_handlers(app):
|
|
| 38 |
"""Setup graceful shutdown handlers."""
|
| 39 |
def shutdown_handler(signum, frame):
|
| 40 |
logger.info("Received shutdown signal, cleaning up...")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 41 |
try:
|
| 42 |
# Save sessions
|
| 43 |
from code.server.session_manager import get_session_manager
|
|
@@ -54,7 +68,7 @@ def setup_signal_handlers(app):
|
|
| 54 |
_unload_model()
|
| 55 |
logger.info("Model unloaded")
|
| 56 |
except Exception as e:
|
| 57 |
-
logger.warning("Failed to unload model: %
|
| 58 |
|
| 59 |
logger.info("Shutdown complete")
|
| 60 |
sys.exit(0)
|
|
@@ -83,15 +97,107 @@ def initialize_session_manager():
|
|
| 83 |
return None
|
| 84 |
|
| 85 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 86 |
def main():
|
| 87 |
start_time = time.time()
|
| 88 |
logger.info("=" * 50)
|
| 89 |
logger.info("SoniCoder v%s starting...", APP_VERSION)
|
| 90 |
logger.info("=" * 50)
|
| 91 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 92 |
# Initialize session manager
|
| 93 |
session_mgr = initialize_session_manager()
|
| 94 |
|
|
|
|
|
|
|
|
|
|
| 95 |
# Start loading default model in background
|
| 96 |
from code.model.loader import start_background_load, add_progress_callback
|
| 97 |
|
|
@@ -137,6 +243,13 @@ def main():
|
|
| 137 |
# Log startup time
|
| 138 |
startup_time = time.time() - start_time
|
| 139 |
logger.info("Startup completed in %.2f seconds", startup_time)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 140 |
logger.info("Launching...")
|
| 141 |
|
| 142 |
# Launch the server
|
|
|
|
| 9 |
- Health check endpoint
|
| 10 |
- Session management initialization
|
| 11 |
- Model optimization on startup
|
| 12 |
+
- **Hugging Face Hub cloud storage integration**
|
| 13 |
+
- Auto-detection for HF Spaces environment
|
| 14 |
|
| 15 |
Blog: https://huggingface.co/blog/introducing-gradio-server
|
| 16 |
"""
|
|
|
|
| 18 |
from __future__ import annotations
|
| 19 |
|
| 20 |
import logging
|
| 21 |
+
import os
|
| 22 |
import signal
|
| 23 |
import sys
|
| 24 |
import time
|
|
|
|
| 41 |
"""Setup graceful shutdown handlers."""
|
| 42 |
def shutdown_handler(signum, frame):
|
| 43 |
logger.info("Received shutdown signal, cleaning up...")
|
| 44 |
+
|
| 45 |
+
# Sync to Hub before shutdown if enabled
|
| 46 |
+
try:
|
| 47 |
+
from code.hub_storage import is_hub_available, get_hub_storage
|
| 48 |
+
if is_hub_available():
|
| 49 |
+
hub = get_hub_storage()
|
| 50 |
+
hub.sync_to_hub(commit_message="Auto-sync before shutdown")
|
| 51 |
+
logger.info("Workspace synced to Hub before shutdown")
|
| 52 |
+
except Exception as e:
|
| 53 |
+
logger.warning("Failed to sync to Hub: %s", e)
|
| 54 |
+
|
| 55 |
try:
|
| 56 |
# Save sessions
|
| 57 |
from code.server.session_manager import get_session_manager
|
|
|
|
| 68 |
_unload_model()
|
| 69 |
logger.info("Model unloaded")
|
| 70 |
except Exception as e:
|
| 71 |
+
logger.warning("Failed to unload model: %e", e)
|
| 72 |
|
| 73 |
logger.info("Shutdown complete")
|
| 74 |
sys.exit(0)
|
|
|
|
| 97 |
return None
|
| 98 |
|
| 99 |
|
| 100 |
+
def initialize_hub_storage():
|
| 101 |
+
"""
|
| 102 |
+
Initialize Hugging Face Hub cloud storage.
|
| 103 |
+
|
| 104 |
+
Auto-detects environment:
|
| 105 |
+
- On HF Spaces: Uses SPACE_ID, HF_TOKEN env vars
|
| 106 |
+
- Local: Requires explicit configuration or HF_TOKEN
|
| 107 |
+
|
| 108 |
+
Returns:
|
| 109 |
+
Initialized HubStorage instance or None if not configured.
|
| 110 |
+
"""
|
| 111 |
+
try:
|
| 112 |
+
from code.hub_storage import init_hub_storage, is_hub_available
|
| 113 |
+
|
| 114 |
+
# Get configuration from environment
|
| 115 |
+
token = os.environ.get("HF_TOKEN", "") or os.environ.get("HF_AUTH_TOKEN", "")
|
| 116 |
+
space_id = os.environ.get("SPACE_ID", "")
|
| 117 |
+
|
| 118 |
+
# Determine mode based on environment
|
| 119 |
+
if space_id:
|
| 120 |
+
# Running on HF Spaces
|
| 121 |
+
logger.info("Detected HF Spaces environment: %s", space_id)
|
| 122 |
+
|
| 123 |
+
# Extract repo_id from space_id (format: "user/space-name")
|
| 124 |
+
if space_id and token:
|
| 125 |
+
# Use a dataset repo for workspace persistence
|
| 126 |
+
# This allows the AI's code to persist across Space restarts
|
| 127 |
+
user_name = space_id.split("/")[0] if "/" in space_id else "unknown"
|
| 128 |
+
dataset_repo_id = os.environ.get(
|
| 129 |
+
"HUB_DATASET_REPO",
|
| 130 |
+
f"{user_name}/sonicoder-workspace"
|
| 131 |
+
)
|
| 132 |
+
|
| 133 |
+
mode = os.environ.get("HUB_STORAGE_MODE", "hybrid")
|
| 134 |
+
|
| 135 |
+
storage = init_hub_storage(
|
| 136 |
+
token=token,
|
| 137 |
+
repo_id=space_id,
|
| 138 |
+
dataset_repo_id=dataset_repo_id,
|
| 139 |
+
mode=mode,
|
| 140 |
+
)
|
| 141 |
+
|
| 142 |
+
stats = storage.get_storage_stats()
|
| 143 |
+
logger.info(
|
| 144 |
+
"Hub storage initialized: mode=%s, hub=%s, dataset=%s",
|
| 145 |
+
stats["mode"],
|
| 146 |
+
stats["hub_enabled"],
|
| 147 |
+
stats["dataset_enabled"]
|
| 148 |
+
)
|
| 149 |
+
|
| 150 |
+
return storage
|
| 151 |
+
else:
|
| 152 |
+
logger.warning("HF Spaces detected but no token provided, using local storage only")
|
| 153 |
+
return None
|
| 154 |
+
elif token:
|
| 155 |
+
# Local mode with token
|
| 156 |
+
repo_id = os.environ.get("HUB_REPO_ID", "")
|
| 157 |
+
dataset_repo_id = os.environ.get("HUB_DATASET_REPO", "")
|
| 158 |
+
|
| 159 |
+
if repo_id or dataset_repo_id:
|
| 160 |
+
storage = init_hub_storage(
|
| 161 |
+
token=token,
|
| 162 |
+
repo_id=repo_id,
|
| 163 |
+
dataset_repo_id=dataset_repo_id,
|
| 164 |
+
mode="hybrid",
|
| 165 |
+
)
|
| 166 |
+
|
| 167 |
+
logger.info("Hub storage initialized in local mode")
|
| 168 |
+
return storage
|
| 169 |
+
|
| 170 |
+
# No configuration available
|
| 171 |
+
logger.info("Hub storage not configured (set HF_TOKEN to enable)")
|
| 172 |
+
return None
|
| 173 |
+
|
| 174 |
+
except ImportError:
|
| 175 |
+
logger.debug("huggingface_hub not installed, skipping Hub storage")
|
| 176 |
+
return None
|
| 177 |
+
except Exception as exc:
|
| 178 |
+
logger.warning("Hub storage initialization failed: %s", exc)
|
| 179 |
+
return None
|
| 180 |
+
|
| 181 |
+
|
| 182 |
def main():
|
| 183 |
start_time = time.time()
|
| 184 |
logger.info("=" * 50)
|
| 185 |
logger.info("SoniCoder v%s starting...", APP_VERSION)
|
| 186 |
logger.info("=" * 50)
|
| 187 |
|
| 188 |
+
# Detect and log environment
|
| 189 |
+
space_id = os.environ.get("SPACE_ID", None)
|
| 190 |
+
if space_id:
|
| 191 |
+
logger.info("Running on Hugging Face Spaces: %s", space_id)
|
| 192 |
+
else:
|
| 193 |
+
logger.info("Running in local mode")
|
| 194 |
+
|
| 195 |
# Initialize session manager
|
| 196 |
session_mgr = initialize_session_manager()
|
| 197 |
|
| 198 |
+
# Initialize Hub storage (cloud persistence)
|
| 199 |
+
hub_storage = initialize_hub_storage()
|
| 200 |
+
|
| 201 |
# Start loading default model in background
|
| 202 |
from code.model.loader import start_background_load, add_progress_callback
|
| 203 |
|
|
|
|
| 243 |
# Log startup time
|
| 244 |
startup_time = time.time() - start_time
|
| 245 |
logger.info("Startup completed in %.2f seconds", startup_time)
|
| 246 |
+
|
| 247 |
+
# Log storage info
|
| 248 |
+
if hub_storage:
|
| 249 |
+
logger.info("Cloud storage: ENABLED (files will persist on HF Hub)")
|
| 250 |
+
else:
|
| 251 |
+
logger.info("Cloud storage: DISABLED (local files only)")
|
| 252 |
+
|
| 253 |
logger.info("Launching...")
|
| 254 |
|
| 255 |
# Launch the server
|
|
@@ -0,0 +1,933 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Hugging Face Hub Cloud Storage Integration.
|
| 2 |
+
|
| 3 |
+
This module provides seamless integration with Hugging Face Hub for:
|
| 4 |
+
- Cloud-based file storage and retrieval
|
| 5 |
+
- Persistent workspace on HF Spaces
|
| 6 |
+
- Direct code writing to repositories
|
| 7 |
+
- Dataset-backed persistence
|
| 8 |
+
|
| 9 |
+
Architecture:
|
| 10 |
+
┌─────────────┐ ┌──────────────────┐ ┌─────────────────┐
|
| 11 |
+
│ AI Agent │────▶│ hub_storage.py │────▶│ HuggingFace Hub │
|
| 12 |
+
│ │ │ (this module) │ │ (Cloud Storage) │
|
| 13 |
+
└─────────────┘ └──────────────────┘ └─────────────────┘
|
| 14 |
+
│
|
| 15 |
+
▼
|
| 16 |
+
┌──────────────────┐
|
| 17 |
+
│ Local Fallback │
|
| 18 |
+
│ (when offline) │
|
| 19 |
+
└──────────────────┘
|
| 20 |
+
|
| 21 |
+
Usage Modes:
|
| 22 |
+
1. LOCAL: Files stored in ./workspace/ (default)
|
| 23 |
+
2. HUB: Files synced to HF Hub repository
|
| 24 |
+
3. HYBRID: Local + Hub sync (recommended for Spaces)
|
| 25 |
+
"""
|
| 26 |
+
|
| 27 |
+
from __future__ import annotations
|
| 28 |
+
|
| 29 |
+
import json
|
| 30 |
+
import logging
|
| 31 |
+
import os
|
| 32 |
+
import tempfile
|
| 33 |
+
import time
|
| 34 |
+
from dataclasses import dataclass, field
|
| 35 |
+
from pathlib import Path
|
| 36 |
+
from typing import Any, Callable, Optional, Iterator
|
| 37 |
+
from contextlib import contextmanager
|
| 38 |
+
|
| 39 |
+
logger = logging.getLogger(__name__)
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
@dataclass
|
| 43 |
+
class HubConfig:
|
| 44 |
+
"""Configuration for HF Hub storage."""
|
| 45 |
+
|
| 46 |
+
# Repository settings
|
| 47 |
+
repo_id: str = "" # e.g., "username/repo-name" or "username/workspace-data"
|
| 48 |
+
repo_type: str = "space" # "space", "model", or "dataset"
|
| 49 |
+
|
| 50 |
+
# Authentication
|
| 51 |
+
token: str = ""
|
| 52 |
+
|
| 53 |
+
# Storage mode
|
| 54 |
+
mode: str = "local" # "local", "hub", or "hybrid"
|
| 55 |
+
|
| 56 |
+
# Workspace path (for local/hybrid)
|
| 57 |
+
local_workspace: str = "./workspace"
|
| 58 |
+
|
| 59 |
+
# Sync settings
|
| 60 |
+
auto_sync: bool = True
|
| 61 |
+
sync_interval: int = 60 # seconds
|
| 62 |
+
|
| 63 |
+
# Dataset repo for persistent storage (recommended for Spaces)
|
| 64 |
+
dataset_repo_id: str = "" # e.g., "username/sonicoder-workspace"
|
| 65 |
+
|
| 66 |
+
def is_hub_enabled(self) -> bool:
|
| 67 |
+
"""Check if Hub storage is enabled."""
|
| 68 |
+
return self.mode in ("hub", "hybrid") and bool(self.repo_id) and bool(self.token)
|
| 69 |
+
|
| 70 |
+
def is_dataset_storage_enabled(self) -> bool:
|
| 71 |
+
"""Check if dataset-based storage is enabled."""
|
| 72 |
+
return bool(self.dataset_repo_id) and bool(self.token)
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
@dataclass
|
| 76 |
+
class FileMetadata:
|
| 77 |
+
"""Metadata for a file in storage."""
|
| 78 |
+
path: str
|
| 79 |
+
size: int = 0
|
| 80 |
+
modified_at: float = field(default_factory=time.time)
|
| 81 |
+
created_at: float = field(default_factory=time.time)
|
| 82 |
+
content_hash: str = ""
|
| 83 |
+
is_local: bool = True
|
| 84 |
+
is_remote: bool = False
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
class HubStorageError(Exception):
|
| 88 |
+
"""Custom exception for Hub storage operations."""
|
| 89 |
+
|
| 90 |
+
def __init__(self, message: str, operation: str = "", recoverable: bool = True):
|
| 91 |
+
self.message = message
|
| 92 |
+
self.operation = operation
|
| 93 |
+
self.recoverable = recoverable
|
| 94 |
+
super().__init__(f"[{operation}] {message}")
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
class HubStorage:
|
| 98 |
+
"""
|
| 99 |
+
Main class for Hugging Face Hub cloud storage integration.
|
| 100 |
+
|
| 101 |
+
Provides transparent file operations that work with both
|
| 102 |
+
local filesystem and HF Hub, with automatic sync.
|
| 103 |
+
|
| 104 |
+
Example:
|
| 105 |
+
>>> storage = HubStorage(
|
| 106 |
+
... repo_id="sonic-coder/sonicoder",
|
| 107 |
+
... token="hf_...",
|
| 108 |
+
... mode="hybrid",
|
| 109 |
+
... dataset_repo_id="sonic-coder/sonicoder-workspace"
|
| 110 |
+
... )
|
| 111 |
+
>>> storage.initialize()
|
| 112 |
+
>>> storage.write_file("app.py", "print('Hello!')")
|
| 113 |
+
>>> content = storage.read_file("app.py")
|
| 114 |
+
>>> files = storage.list_files()
|
| 115 |
+
"""
|
| 116 |
+
|
| 117 |
+
def __init__(
|
| 118 |
+
self,
|
| 119 |
+
config: Optional[HubConfig] = None,
|
| 120 |
+
**kwargs
|
| 121 |
+
):
|
| 122 |
+
self.config = config or HubConfig(**kwargs)
|
| 123 |
+
self._api = None
|
| 124 |
+
self._initialized = False
|
| 125 |
+
self._local_workspace: Path = Path(self.config.local_workspace)
|
| 126 |
+
self._sync_lock_needed = False
|
| 127 |
+
|
| 128 |
+
# Cache for remote files
|
| 129 |
+
self._remote_cache: dict[str, FileMetadata] = {}
|
| 130 |
+
self._cache_timestamp: float = 0
|
| 131 |
+
self._cache_ttl: int = 30 # seconds
|
| 132 |
+
|
| 133 |
+
# Operation callbacks
|
| 134 |
+
self._on_sync: Optional[Callable] = None
|
| 135 |
+
self._on_error: Optional[Callable] = None
|
| 136 |
+
|
| 137 |
+
@property
|
| 138 |
+
def api(self):
|
| 139 |
+
"""Lazy initialization of HfApi."""
|
| 140 |
+
if self._api is None:
|
| 141 |
+
try:
|
| 142 |
+
from huggingface_hub import HfApi
|
| 143 |
+
self._api = HfApi(token=self.config.token)
|
| 144 |
+
except ImportError:
|
| 145 |
+
raise HubStorageError(
|
| 146 |
+
"huggingface_hub package not installed",
|
| 147 |
+
operation="initialize",
|
| 148 |
+
recoverable=False
|
| 149 |
+
)
|
| 150 |
+
return self._api
|
| 151 |
+
|
| 152 |
+
def initialize(self) -> dict[str, Any]:
|
| 153 |
+
"""
|
| 154 |
+
Initialize the Hub storage system.
|
| 155 |
+
|
| 156 |
+
Creates necessary directories, validates connection,
|
| 157 |
+
and sets up the storage backend.
|
| 158 |
+
|
| 159 |
+
Returns:
|
| 160 |
+
Dict with initialization status and info.
|
| 161 |
+
"""
|
| 162 |
+
result = {
|
| 163 |
+
"success": False,
|
| 164 |
+
"mode": self.config.mode,
|
| 165 |
+
"hub_enabled": False,
|
| 166 |
+
"dataset_enabled": False,
|
| 167 |
+
"workspace_path": str(self._local_workspace),
|
| 168 |
+
}
|
| 169 |
+
|
| 170 |
+
try:
|
| 171 |
+
# Create local workspace directory
|
| 172 |
+
self._local_workspace.mkdir(parents=True, exist_ok=True)
|
| 173 |
+
|
| 174 |
+
# Validate Hub connection if enabled
|
| 175 |
+
if self.config.is_hub_enabled():
|
| 176 |
+
try:
|
| 177 |
+
# Test API connection by getting user info
|
| 178 |
+
user_info = self.api.whoami()
|
| 179 |
+
result["hub_enabled"] = True
|
| 180 |
+
result["user"] = user_info.get("name", "unknown")
|
| 181 |
+
logger.info("Connected to HF Hub as: %s", result["user"])
|
| 182 |
+
|
| 183 |
+
# Ensure repo exists
|
| 184 |
+
self._ensure_repo_exists()
|
| 185 |
+
|
| 186 |
+
except Exception as e:
|
| 187 |
+
logger.warning("Hub connection failed: %s", e)
|
| 188 |
+
result["hub_error"] = str(e)
|
| 189 |
+
# Fall back to local mode
|
| 190 |
+
self.config.mode = "local"
|
| 191 |
+
|
| 192 |
+
# Check dataset storage
|
| 193 |
+
if self.config.is_dataset_storage_enabled():
|
| 194 |
+
try:
|
| 195 |
+
self._ensure_dataset_repo_exists()
|
| 196 |
+
result["dataset_enabled"] = True
|
| 197 |
+
logger.info("Dataset storage ready: %s", self.config.dataset_repo_id)
|
| 198 |
+
except Exception as e:
|
| 199 |
+
logger.warning("Dataset setup failed: %s", e)
|
| 200 |
+
result["dataset_error"] = str(e)
|
| 201 |
+
|
| 202 |
+
self._initialized = True
|
| 203 |
+
result["success"] = True
|
| 204 |
+
|
| 205 |
+
logger.info(
|
| 206 |
+
"HubStorage initialized: mode=%s, hub=%s, dataset=%s",
|
| 207 |
+
self.config.mode,
|
| 208 |
+
result["hub_enabled"],
|
| 209 |
+
result["dataset_enabled"]
|
| 210 |
+
)
|
| 211 |
+
|
| 212 |
+
except Exception as e:
|
| 213 |
+
logger.exception("Failed to initialize HubStorage")
|
| 214 |
+
result["error"] = str(e)
|
| 215 |
+
|
| 216 |
+
return result
|
| 217 |
+
|
| 218 |
+
def _ensure_repo_exists(self) -> None:
|
| 219 |
+
"""Ensure the target repository exists."""
|
| 220 |
+
try:
|
| 221 |
+
self.api.repo_info(
|
| 222 |
+
repo_id=self.config.repo_id,
|
| 223 |
+
repo_type=self.config.repo_type
|
| 224 |
+
)
|
| 225 |
+
except Exception:
|
| 226 |
+
# Repo doesn't exist, create it
|
| 227 |
+
logger.info("Creating repo: %s", self.config.repo_id)
|
| 228 |
+
self.api.create_repo(
|
| 229 |
+
repo_id=self.config.repo_id,
|
| 230 |
+
repo_type=self.config.repo_type,
|
| 231 |
+
exist_ok=True,
|
| 232 |
+
private=False
|
| 233 |
+
)
|
| 234 |
+
|
| 235 |
+
def _ensure_dataset_repo_exists(self) -> None:
|
| 236 |
+
"""Ensure the dataset repository exists for persistent storage."""
|
| 237 |
+
try:
|
| 238 |
+
self.api.repo_info(
|
| 239 |
+
repo_id=self.config.dataset_repo_id,
|
| 240 |
+
repo_type="dataset"
|
| 241 |
+
)
|
| 242 |
+
except Exception:
|
| 243 |
+
logger.info("Creating dataset repo: %s", self.config.dataset_repo_id)
|
| 244 |
+
self.api.create_repo(
|
| 245 |
+
repo_id=self.config.dataset_repo_id,
|
| 246 |
+
repo_type="dataset",
|
| 247 |
+
exist_ok=True,
|
| 248 |
+
private=True # Workspaces are usually private
|
| 249 |
+
)
|
| 250 |
+
|
| 251 |
+
# ─── File Operations ──────────────────────────────────────────────
|
| 252 |
+
|
| 253 |
+
def write_file(
|
| 254 |
+
self,
|
| 255 |
+
path: str,
|
| 256 |
+
content: str | bytes,
|
| 257 |
+
commit_message: str | None = None,
|
| 258 |
+
sync_to_hub: bool = True
|
| 259 |
+
) -> FileMetadata:
|
| 260 |
+
"""
|
| 261 |
+
Write a file to storage.
|
| 262 |
+
|
| 263 |
+
In hybrid/local mode, writes to local filesystem first.
|
| 264 |
+
If sync_to_hub and hub is enabled, also uploads to HF Hub.
|
| 265 |
+
|
| 266 |
+
Args:
|
| 267 |
+
path: Relative file path (e.g., "src/app.py")
|
| 268 |
+
content: File content (string or bytes)
|
| 269 |
+
commit_message: Optional git commit message
|
| 270 |
+
sync_to_hub: Whether to sync to Hub immediately
|
| 271 |
+
|
| 272 |
+
Returns:
|
| 273 |
+
FileMetadata with file information.
|
| 274 |
+
"""
|
| 275 |
+
# Normalize path
|
| 276 |
+
path = self._normalize_path(path)
|
| 277 |
+
|
| 278 |
+
# Generate commit message if not provided
|
| 279 |
+
if not commit_message:
|
| 280 |
+
commit_message = f"Update {path}"
|
| 281 |
+
|
| 282 |
+
# Write to local filesystem first
|
| 283 |
+
local_path = self._local_workspace / path
|
| 284 |
+
local_path.parent.mkdir(parents=True, exist_ok=True)
|
| 285 |
+
|
| 286 |
+
if isinstance(content, bytes):
|
| 287 |
+
local_path.write_bytes(content)
|
| 288 |
+
else:
|
| 289 |
+
local_path.write_text(content, encoding='utf-8')
|
| 290 |
+
|
| 291 |
+
# Create metadata
|
| 292 |
+
metadata = FileMetadata(
|
| 293 |
+
path=path,
|
| 294 |
+
size=local_path.stat().st_size,
|
| 295 |
+
is_local=True
|
| 296 |
+
)
|
| 297 |
+
|
| 298 |
+
# Sync to Hub if enabled
|
| 299 |
+
if sync_to_hub and self._should_sync_to_hub():
|
| 300 |
+
try:
|
| 301 |
+
metadata = self._upload_to_hub(path, content, commit_message)
|
| 302 |
+
metadata.is_remote = True
|
| 303 |
+
except Exception as e:
|
| 304 |
+
logger.warning("Failed to sync %s to Hub: %s", path, e)
|
| 305 |
+
if self._on_error:
|
| 306 |
+
self._on_error("write", path, e)
|
| 307 |
+
|
| 308 |
+
return metadata
|
| 309 |
+
|
| 310 |
+
def read_file(self, path: str, use_cache: bool = True) -> str:
|
| 311 |
+
"""
|
| 312 |
+
Read a file from storage.
|
| 313 |
+
|
| 314 |
+
Tries local first, then falls back to Hub if available.
|
| 315 |
+
|
| 316 |
+
Args:
|
| 317 |
+
path: Relative file path
|
| 318 |
+
use_cache: Whether to use cached remote file list
|
| 319 |
+
|
| 320 |
+
Returns:
|
| 321 |
+
File content as string.
|
| 322 |
+
|
| 323 |
+
Raises:
|
| 324 |
+
HubStorageError if file not found anywhere.
|
| 325 |
+
"""
|
| 326 |
+
path = self._normalize_path(path)
|
| 327 |
+
|
| 328 |
+
# Try local first
|
| 329 |
+
local_path = self._local_workspace / path
|
| 330 |
+
if local_path.exists():
|
| 331 |
+
return local_path.read_text(encoding='utf-8')
|
| 332 |
+
|
| 333 |
+
# Try Hub if enabled
|
| 334 |
+
if self.config.is_hub_enabled() or self.config.is_dataset_storage_enabled():
|
| 335 |
+
try:
|
| 336 |
+
return self._download_from_hub(path)
|
| 337 |
+
except Exception as e:
|
| 338 |
+
logger.debug("Could not download %s from Hub: %s", path, e)
|
| 339 |
+
|
| 340 |
+
raise HubStorageError(
|
| 341 |
+
f"File not found: {path}",
|
| 342 |
+
operation="read_file",
|
| 343 |
+
recoverable=False
|
| 344 |
+
)
|
| 345 |
+
|
| 346 |
+
def read_binary(self, path: str) -> bytes:
|
| 347 |
+
"""Read a file as binary."""
|
| 348 |
+
path = self._normalize_path(path)
|
| 349 |
+
|
| 350 |
+
local_path = self._local_workspace / path
|
| 351 |
+
if local_path.exists():
|
| 352 |
+
return local_path.read_bytes()
|
| 353 |
+
|
| 354 |
+
if self.config.is_hub_enabled():
|
| 355 |
+
# Download to temp and read
|
| 356 |
+
with tempfile.NamedTemporaryFile(delete=False) as tmp:
|
| 357 |
+
tmp_path = tmp.name
|
| 358 |
+
|
| 359 |
+
try:
|
| 360 |
+
self.api.hub_download(
|
| 361 |
+
repo_id=self._get_target_repo(),
|
| 362 |
+
filename=path,
|
| 363 |
+
repo_type=self._get_target_repo_type(),
|
| 364 |
+
local_dir=os.path.dirname(tmp_path),
|
| 365 |
+
)
|
| 366 |
+
|
| 367 |
+
# Find downloaded file
|
| 368 |
+
downloaded = Path(tmp_path).parent / Path(path).name
|
| 369 |
+
if downloaded.exists():
|
| 370 |
+
return downloaded.read_bytes()
|
| 371 |
+
finally:
|
| 372 |
+
# Cleanup temp
|
| 373 |
+
if os.path.exists(tmp_path):
|
| 374 |
+
os.unlink(tmp_path)
|
| 375 |
+
|
| 376 |
+
raise HubStorageError(f"Binary file not found: {path}", operation="read_binary")
|
| 377 |
+
|
| 378 |
+
def delete_file(self, path: str, commit_message: str | None = None) -> bool:
|
| 379 |
+
"""
|
| 380 |
+
Delete a file from storage.
|
| 381 |
+
|
| 382 |
+
Args:
|
| 383 |
+
path: Relative file path
|
| 384 |
+
commit_message: Optional commit message
|
| 385 |
+
|
| 386 |
+
Returns:
|
| 387 |
+
True if deleted successfully.
|
| 388 |
+
"""
|
| 389 |
+
path = self._normalize_path(path)
|
| 390 |
+
|
| 391 |
+
if not commit_message:
|
| 392 |
+
commit_message = f"Delete {path}"
|
| 393 |
+
|
| 394 |
+
success = True
|
| 395 |
+
|
| 396 |
+
# Delete from local
|
| 397 |
+
local_path = self._local_workspace / path
|
| 398 |
+
if local_path.exists():
|
| 399 |
+
local_path.unlink()
|
| 400 |
+
# Remove empty parent directories
|
| 401 |
+
self._cleanup_empty_dirs(local_path.parent)
|
| 402 |
+
|
| 403 |
+
# Delete from Hub
|
| 404 |
+
if self._should_sync_to_hub():
|
| 405 |
+
try:
|
| 406 |
+
self.api.delete_file(
|
| 407 |
+
path_in_repo=path,
|
| 408 |
+
repo_id=self._get_target_repo(),
|
| 409 |
+
repo_type=self._get_target_repo_type(),
|
| 410 |
+
commit_message=commit_message
|
| 411 |
+
)
|
| 412 |
+
except Exception as e:
|
| 413 |
+
logger.warning("Failed to delete %s from Hub: %s", path, e)
|
| 414 |
+
success = False
|
| 415 |
+
|
| 416 |
+
return success
|
| 417 |
+
|
| 418 |
+
def file_exists(self, path: str) -> bool:
|
| 419 |
+
"""Check if a file exists in storage."""
|
| 420 |
+
path = self._normalize_path(path)
|
| 421 |
+
|
| 422 |
+
# Check local
|
| 423 |
+
if (self._local_workspace / path).exists():
|
| 424 |
+
return True
|
| 425 |
+
|
| 426 |
+
# Check Hub cache or remote
|
| 427 |
+
if self.config.is_hub_enabled():
|
| 428 |
+
try:
|
| 429 |
+
files = self.list_files(use_cache=True)
|
| 430 |
+
return any(f["path"] == path for f in files)
|
| 431 |
+
except Exception:
|
| 432 |
+
pass
|
| 433 |
+
|
| 434 |
+
return False
|
| 435 |
+
|
| 436 |
+
def list_files(
|
| 437 |
+
self,
|
| 438 |
+
path: str = ".",
|
| 439 |
+
recursive: bool = True,
|
| 440 |
+
use_cache: bool = True
|
| 441 |
+
) -> list[dict[str, Any]]:
|
| 442 |
+
"""
|
| 443 |
+
List files in storage.
|
| 444 |
+
|
| 445 |
+
Combines local and remote file listings.
|
| 446 |
+
|
| 447 |
+
Args:
|
| 448 |
+
path: Directory path to list (default: root)
|
| 449 |
+
recursive: Whether to list recursively
|
| 450 |
+
use_cache: Whether to use cached file listing
|
| 451 |
+
|
| 452 |
+
Returns:
|
| 453 |
+
List of file info dicts with 'path', 'size', 'modified' keys.
|
| 454 |
+
"""
|
| 455 |
+
files = []
|
| 456 |
+
seen_paths = set()
|
| 457 |
+
|
| 458 |
+
# List local files
|
| 459 |
+
base_path = self._local_workspace / path
|
| 460 |
+
if base_path.exists():
|
| 461 |
+
for file_path in base_path.rglob("*") if recursive else base_path.iterdir():
|
| 462 |
+
if file_path.is_file():
|
| 463 |
+
rel_path = str(file_path.relative_to(self._local_workspace))
|
| 464 |
+
stat = file_path.stat()
|
| 465 |
+
files.append({
|
| 466 |
+
"path": rel_path.replace(os.sep, "/"),
|
| 467 |
+
"size": stat.st_size,
|
| 468 |
+
"modified": stat.st_mtime,
|
| 469 |
+
"is_local": True,
|
| 470 |
+
"is_remote": False,
|
| 471 |
+
})
|
| 472 |
+
seen_paths.add(rel_path)
|
| 473 |
+
|
| 474 |
+
# List remote files if hub enabled
|
| 475 |
+
if self.config.is_hub_enabled() or self.config.is_dataset_storage_enabled():
|
| 476 |
+
try:
|
| 477 |
+
remote_files = self._list_hub_files(use_cache=use_cache)
|
| 478 |
+
for rf in remote_files:
|
| 479 |
+
if rf["path"] not in seen_paths:
|
| 480 |
+
rf["is_local"] = False
|
| 481 |
+
rf["is_remote"] = True
|
| 482 |
+
files.append(rf)
|
| 483 |
+
except Exception as e:
|
| 484 |
+
logger.warning("Failed to list remote files: %e", e)
|
| 485 |
+
|
| 486 |
+
# Sort by path
|
| 487 |
+
files.sort(key=lambda x: x["path"])
|
| 488 |
+
|
| 489 |
+
return files
|
| 490 |
+
|
| 491 |
+
# ─── Hub Operations ────────────────────────────────────────────────
|
| 492 |
+
|
| 493 |
+
def _should_sync_to_hub(self) -> bool:
|
| 494 |
+
"""Check if we should sync to Hub based on config."""
|
| 495 |
+
return (
|
| 496 |
+
self.config.is_hub_enabled() or
|
| 497 |
+
self.config.is_dataset_storage_enabled()
|
| 498 |
+
) and self.config.auto_sync
|
| 499 |
+
|
| 500 |
+
def _get_target_repo(self) -> str:
|
| 501 |
+
"""Get the target repository ID for operations."""
|
| 502 |
+
# Prefer dataset repo for workspace files
|
| 503 |
+
if self.config.is_dataset_storage_enabled():
|
| 504 |
+
return self.config.dataset_repo_id
|
| 505 |
+
return self.config.repo_id
|
| 506 |
+
|
| 507 |
+
def _get_target_repo_type(self) -> str:
|
| 508 |
+
"""Get the target repository type."""
|
| 509 |
+
if self.config.is_dataset_storage_enabled():
|
| 510 |
+
return "dataset"
|
| 511 |
+
return self.config.repo_type
|
| 512 |
+
|
| 513 |
+
def _upload_to_hub(
|
| 514 |
+
self,
|
| 515 |
+
path: str,
|
| 516 |
+
content: str | bytes,
|
| 517 |
+
commit_message: str
|
| 518 |
+
) -> FileMetadata:
|
| 519 |
+
"""Upload a file to HF Hub."""
|
| 520 |
+
|
| 521 |
+
# For large content, use temp file
|
| 522 |
+
if isinstance(content, str) and len(content) > 100000: # > 100KB
|
| 523 |
+
with tempfile.NamedTemporaryFile(
|
| 524 |
+
mode='w',
|
| 525 |
+
suffix=Path(path).suffix,
|
| 526 |
+
delete=False,
|
| 527 |
+
encoding='utf-8'
|
| 528 |
+
) as tmp:
|
| 529 |
+
tmp.write(content)
|
| 530 |
+
tmp_path = tmp.name
|
| 531 |
+
|
| 532 |
+
try:
|
| 533 |
+
self.api.upload_file(
|
| 534 |
+
path_or_fileobj=tmp_path,
|
| 535 |
+
path_in_repo=path,
|
| 536 |
+
repo_id=self._get_target_repo(),
|
| 537 |
+
repo_type=self._get_target_repo_type(),
|
| 538 |
+
commit_message=commit_message
|
| 539 |
+
)
|
| 540 |
+
finally:
|
| 541 |
+
os.unlink(tmp_path)
|
| 542 |
+
else:
|
| 543 |
+
# Upload directly from content
|
| 544 |
+
content_bytes = content.encode('utf-8') if isinstance(content, str) else content
|
| 545 |
+
|
| 546 |
+
from io import BytesIO
|
| 547 |
+
self.api.upload_file(
|
| 548 |
+
path_or_fileobj=BytesIO(content_bytes),
|
| 549 |
+
path_in_repo=path,
|
| 550 |
+
repo_id=self._get_target_repo(),
|
| 551 |
+
repo_type=self._get_target_repo_type(),
|
| 552 |
+
commit_message=commit_message
|
| 553 |
+
)
|
| 554 |
+
|
| 555 |
+
logger.info("Uploaded %s to Hub (%s)", path, self._get_target_repo())
|
| 556 |
+
|
| 557 |
+
return FileMetadata(
|
| 558 |
+
path=path,
|
| 559 |
+
size=len(content) if isinstance(content, str) else len(content_bytes),
|
| 560 |
+
is_remote=True,
|
| 561 |
+
is_local=(self._local_workspace / path).exists()
|
| 562 |
+
)
|
| 563 |
+
|
| 564 |
+
def _download_from_hub(self, path: str) -> str:
|
| 565 |
+
"""Download a file from HF Hub."""
|
| 566 |
+
|
| 567 |
+
# Use hf_hub_download for caching
|
| 568 |
+
from huggingface_hub import hf_hub_download
|
| 569 |
+
|
| 570 |
+
local_path = hf_hub_download(
|
| 571 |
+
repo_id=self._get_target_repo(),
|
| 572 |
+
filename=path,
|
| 573 |
+
repo_type=self._get_target_repo_type(),
|
| 574 |
+
token=self.config.token
|
| 575 |
+
)
|
| 576 |
+
|
| 577 |
+
# Read and return content
|
| 578 |
+
with open(local_path, 'r', encoding='utf-8') as f:
|
| 579 |
+
return f.read()
|
| 580 |
+
|
| 581 |
+
def _list_hub_files(
|
| 582 |
+
self,
|
| 583 |
+
use_cache: bool = True
|
| 584 |
+
) -> list[dict[str, Any]]:
|
| 585 |
+
"""List files in the Hub repository."""
|
| 586 |
+
|
| 587 |
+
# Check cache
|
| 588 |
+
if use_cache and self._remote_cache:
|
| 589 |
+
if time.time() - self._cache_timestamp < self._cache_ttl:
|
| 590 |
+
return list(self._remote_cache.values())
|
| 591 |
+
|
| 592 |
+
# Fetch from API
|
| 593 |
+
repo_info = self.api.repo_info(
|
| 594 |
+
repo_id=self._get_target_repo(),
|
| 595 |
+
repo_type=self._get_target_repo_type()
|
| 596 |
+
)
|
| 597 |
+
|
| 598 |
+
files = []
|
| 599 |
+
siblings = getattr(repo_info, 'siblings', [])
|
| 600 |
+
|
| 601 |
+
for sibling in siblings:
|
| 602 |
+
if hasattr(sibling, 'rfilename'):
|
| 603 |
+
files.append({
|
| 604 |
+
"path": sibling.rfilename,
|
| 605 |
+
"size": getattr(sibling, 'size', 0),
|
| 606 |
+
"blob_id": getattr(sibling, 'blob_id', ''),
|
| 607 |
+
"is_local": False,
|
| 608 |
+
"is_remote": True,
|
| 609 |
+
})
|
| 610 |
+
|
| 611 |
+
# Update cache
|
| 612 |
+
self._remote_cache = {f["path"]: f for f in files}
|
| 613 |
+
self._cache_timestamp = time.time()
|
| 614 |
+
|
| 615 |
+
return files
|
| 616 |
+
|
| 617 |
+
# ─── Sync Operations ──────────────────────────────────────────────
|
| 618 |
+
|
| 619 |
+
def sync_to_hub(
|
| 620 |
+
self,
|
| 621 |
+
path: str | None = None,
|
| 622 |
+
commit_message: str = "Sync workspace"
|
| 623 |
+
) -> dict[str, Any]:
|
| 624 |
+
"""
|
| 625 |
+
Sync local files to Hub.
|
| 626 |
+
|
| 627 |
+
Args:
|
| 628 |
+
path: Specific file or directory to sync (None = all)
|
| 629 |
+
commit_message: Commit message for changes
|
| 630 |
+
|
| 631 |
+
Returns:
|
| 632 |
+
Dict with sync results.
|
| 633 |
+
"""
|
| 634 |
+
result = {
|
| 635 |
+
"success": False,
|
| 636 |
+
"uploaded": [],
|
| 637 |
+
"skipped": [],
|
| 638 |
+
"errors": []
|
| 639 |
+
}
|
| 640 |
+
|
| 641 |
+
if not self.config.is_hub_enabled() and not self.config.is_dataset_storage_enabled():
|
| 642 |
+
result["errors"].append("Hub storage not configured")
|
| 643 |
+
return result
|
| 644 |
+
|
| 645 |
+
try:
|
| 646 |
+
files_to_sync = []
|
| 647 |
+
|
| 648 |
+
if path:
|
| 649 |
+
# Sync specific file/directory
|
| 650 |
+
full_path = self._local_workspace / path
|
| 651 |
+
if full_path.is_file():
|
| 652 |
+
files_to_sync.append(path)
|
| 653 |
+
elif full_path.is_dir():
|
| 654 |
+
for fp in full_path.rglob("*"):
|
| 655 |
+
if fp.is_file():
|
| 656 |
+
files_to_sync.append(str(fp.relative_to(self._local_workspace)))
|
| 657 |
+
else:
|
| 658 |
+
# Sync all local files
|
| 659 |
+
for fp in self._local_workspace.rglob("*"):
|
| 660 |
+
if fp.is_file():
|
| 661 |
+
rel = str(fp.relative_to(self._local_workspace))
|
| 662 |
+
files_to_sync.append(rel)
|
| 663 |
+
|
| 664 |
+
# Upload each file
|
| 665 |
+
for file_path in files_to_sync:
|
| 666 |
+
try:
|
| 667 |
+
local_path = self._local_workspace / file_path
|
| 668 |
+
content = local_path.read_text(encoding='utf-8')
|
| 669 |
+
|
| 670 |
+
self._upload_to_hub(
|
| 671 |
+
file_path,
|
| 672 |
+
content,
|
| 673 |
+
f"{commit_message}: {file_path}"
|
| 674 |
+
)
|
| 675 |
+
|
| 676 |
+
result["uploaded"].append(file_path)
|
| 677 |
+
|
| 678 |
+
except Exception as e:
|
| 679 |
+
logger.error("Failed to sync %s: %s", file_path, e)
|
| 680 |
+
result["errors"].append({"file": file_path, "error": str(e)})
|
| 681 |
+
|
| 682 |
+
result["success"] = len(result["errors"]) == 0
|
| 683 |
+
|
| 684 |
+
if self._on_sync:
|
| 685 |
+
self._on_sync(result)
|
| 686 |
+
|
| 687 |
+
except Exception as e:
|
| 688 |
+
result["errors"].append({"file": "*", "error": str(e)})
|
| 689 |
+
logger.exception("Sync failed")
|
| 690 |
+
|
| 691 |
+
return result
|
| 692 |
+
|
| 693 |
+
def sync_from_hub(
|
| 694 |
+
self,
|
| 695 |
+
path: str | None = None
|
| 696 |
+
) -> dict[str, Any]:
|
| 697 |
+
"""
|
| 698 |
+
Sync files from Hub to local.
|
| 699 |
+
|
| 700 |
+
Args:
|
| 701 |
+
path: Specific file to download (None = all)
|
| 702 |
+
|
| 703 |
+
Returns:
|
| 704 |
+
Dict with sync results.
|
| 705 |
+
"""
|
| 706 |
+
result = {
|
| 707 |
+
"success": False,
|
| 708 |
+
"downloaded": [],
|
| 709 |
+
"errors": []
|
| 710 |
+
}
|
| 711 |
+
|
| 712 |
+
try:
|
| 713 |
+
if path:
|
| 714 |
+
# Download specific file
|
| 715 |
+
content = self._download_from_hub(path)
|
| 716 |
+
local_path = self._local_workspace / path
|
| 717 |
+
local_path.parent.mkdir(parents=True, exist_ok=True)
|
| 718 |
+
local_path.write_text(content, encoding='utf-8')
|
| 719 |
+
result["downloaded"].append(path)
|
| 720 |
+
else:
|
| 721 |
+
# Download all files
|
| 722 |
+
remote_files = self._list_hub_files(use_cache=False)
|
| 723 |
+
|
| 724 |
+
for rf in remote_files:
|
| 725 |
+
try:
|
| 726 |
+
content = self._download_from_hub(rf["path"])
|
| 727 |
+
local_path = self._local_workspace / rf["path"]
|
| 728 |
+
local_path.parent.mkdir(parents=True, exist_ok=True)
|
| 729 |
+
local_path.write_text(content, encoding='utf-8')
|
| 730 |
+
result["downloaded"].append(rf["path"])
|
| 731 |
+
except Exception as e:
|
| 732 |
+
result["errors"].append({
|
| 733 |
+
"file": rf["path"],
|
| 734 |
+
"error": str(e)
|
| 735 |
+
})
|
| 736 |
+
|
| 737 |
+
result["success"] = len(result["errors"]) == len(remote_files) == 0 or len(remote_files) == 0
|
| 738 |
+
|
| 739 |
+
except Exception as e:
|
| 740 |
+
result["errors"].append({"file": "*", "error": str(e)})
|
| 741 |
+
logger.exception("Sync from hub failed")
|
| 742 |
+
|
| 743 |
+
return result
|
| 744 |
+
|
| 745 |
+
def full_sync(self) -> dict[str, Any]:
|
| 746 |
+
"""
|
| 747 |
+
Perform bidirectional sync between local and Hub.
|
| 748 |
+
|
| 749 |
+
Merges changes from both sides.
|
| 750 |
+
"""
|
| 751 |
+
result = {
|
| 752 |
+
"to_hub": {},
|
| 753 |
+
"from_hub": {}
|
| 754 |
+
}
|
| 755 |
+
|
| 756 |
+
# Push local to hub
|
| 757 |
+
result["to_hub"] = self.sync_to_hub(commit_message="Full sync push")
|
| 758 |
+
|
| 759 |
+
# Pull new/updated from hub
|
| 760 |
+
result["from_hub"] = self.sync_from_hub()
|
| 761 |
+
|
| 762 |
+
return result
|
| 763 |
+
|
| 764 |
+
# ─── Utility Methods ──────────────────────────────────────────────
|
| 765 |
+
|
| 766 |
+
def _normalize_path(self, path: str) -> str:
|
| 767 |
+
"""Normalize a file path."""
|
| 768 |
+
# Remove leading ./
|
| 769 |
+
if path.startswith("./"):
|
| 770 |
+
path = path[2:]
|
| 771 |
+
# Convert backslashes
|
| 772 |
+
path = path.replace("\\", "/")
|
| 773 |
+
# Remove double slashes
|
| 774 |
+
while "//" in path:
|
| 775 |
+
path = path.replace("//", "/")
|
| 776 |
+
# Security: prevent path traversal
|
| 777 |
+
if ".." in path:
|
| 778 |
+
raise HubStorageError(
|
| 779 |
+
"Path traversal detected",
|
| 780 |
+
operation="_normalize_path",
|
| 781 |
+
recoverable=False
|
| 782 |
+
)
|
| 783 |
+
return path.lstrip("/")
|
| 784 |
+
|
| 785 |
+
def _cleanup_empty_dirs(self, dir_path: Path) -> None:
|
| 786 |
+
"""Remove empty directories up to workspace root."""
|
| 787 |
+
try:
|
| 788 |
+
while dir_path != self._local_workspace:
|
| 789 |
+
if dir_path.is_dir() and not any(dir_path.iterdir()):
|
| 790 |
+
dir_path.rmdir()
|
| 791 |
+
dir_path = dir_path.parent
|
| 792 |
+
else:
|
| 793 |
+
break
|
| 794 |
+
except Exception:
|
| 795 |
+
pass
|
| 796 |
+
|
| 797 |
+
def get_storage_stats(self) -> dict[str, Any]:
|
| 798 |
+
"""Get statistics about current storage."""
|
| 799 |
+
stats = {
|
| 800 |
+
"mode": self.config.mode,
|
| 801 |
+
"hub_enabled": self.config.is_hub_enabled(),
|
| 802 |
+
"dataset_enabled": self.config.is_dataset_storage_enabled(),
|
| 803 |
+
"local_files": 0,
|
| 804 |
+
"local_size_mb": 0,
|
| 805 |
+
"last_sync": self._cache_timestamp,
|
| 806 |
+
}
|
| 807 |
+
|
| 808 |
+
# Count local files
|
| 809 |
+
if self._local_workspace.exists():
|
| 810 |
+
for f in self._local_workspace.rglob("*"):
|
| 811 |
+
if f.is_file():
|
| 812 |
+
stats["local_files"] += 1
|
| 813 |
+
stats["local_size_mb"] += f.stat().st_size / (1024 * 1024)
|
| 814 |
+
|
| 815 |
+
stats["local_size_mb"] = round(stats["local_size_mb"], 2)
|
| 816 |
+
|
| 817 |
+
return stats
|
| 818 |
+
|
| 819 |
+
def clear_cache(self) -> None:
|
| 820 |
+
"""Clear the remote file cache."""
|
| 821 |
+
self._remote_cache = {}
|
| 822 |
+
self._cache_timestamp = 0
|
| 823 |
+
|
| 824 |
+
def set_callbacks(
|
| 825 |
+
self,
|
| 826 |
+
on_sync: Callable | None = None,
|
| 827 |
+
on_error: Callable | None = None
|
| 828 |
+
) -> None:
|
| 829 |
+
"""Set callback functions for events."""
|
| 830 |
+
self._on_sync = on_sync
|
| 831 |
+
self._on_error = on_error
|
| 832 |
+
|
| 833 |
+
@contextmanager
|
| 834 |
+
def batch_operation(self, commit_message: str = "Batch update"):
|
| 835 |
+
"""
|
| 836 |
+
Context manager for batch operations.
|
| 837 |
+
|
| 838 |
+
All writes within the context are grouped into one commit.
|
| 839 |
+
|
| 840 |
+
Usage:
|
| 841 |
+
with storage.batch_operation("Create API"):
|
| 842 |
+
storage.write_file("app.py", "...")
|
| 843 |
+
storage.write_file("config.py", "...")
|
| 844 |
+
"""
|
| 845 |
+
original_auto_sync = self.config.auto_sync
|
| 846 |
+
self.config.auto_sync = False
|
| 847 |
+
|
| 848 |
+
files_written = []
|
| 849 |
+
|
| 850 |
+
try:
|
| 851 |
+
yield files_written
|
| 852 |
+
|
| 853 |
+
# Commit all at once
|
| 854 |
+
if files_written:
|
| 855 |
+
self.sync_to_hub(commit_message=commit_message)
|
| 856 |
+
finally:
|
| 857 |
+
self.config.auto_sync = original_auto_sync
|
| 858 |
+
|
| 859 |
+
|
| 860 |
+
# ─── Global Instance Management ────────────────────────────────────────
|
| 861 |
+
|
| 862 |
+
_global_storage: Optional[HubStorage] = None
|
| 863 |
+
|
| 864 |
+
|
| 865 |
+
def get_hub_storage(**kwargs) -> HubStorage:
|
| 866 |
+
"""Get or create the global HubStorage instance."""
|
| 867 |
+
global _global_storage
|
| 868 |
+
if _global_storage is None:
|
| 869 |
+
_global_storage = HubStorage(**kwargs)
|
| 870 |
+
return _global_storage
|
| 871 |
+
|
| 872 |
+
|
| 873 |
+
def init_hub_storage(
|
| 874 |
+
token: str = "",
|
| 875 |
+
repo_id: str = "",
|
| 876 |
+
dataset_repo_id: str = "",
|
| 877 |
+
mode: str = "auto",
|
| 878 |
+
**kwargs
|
| 879 |
+
) -> HubStorage:
|
| 880 |
+
"""
|
| 881 |
+
Initialize the global HubStorage instance.
|
| 882 |
+
|
| 883 |
+
Auto-detects environment when running on HF Spaces.
|
| 884 |
+
|
| 885 |
+
Args:
|
| 886 |
+
token: HF API token
|
| 887 |
+
repo_id: Main repository ID
|
| 888 |
+
dataset_repo_id: Dataset repository for workspace storage
|
| 889 |
+
mode: Storage mode ("local", "hub", "hybrid", "auto")
|
| 890 |
+
|
| 891 |
+
Returns:
|
| 892 |
+
Initialized HubStorage instance.
|
| 893 |
+
"""
|
| 894 |
+
global _global_storage
|
| 895 |
+
|
| 896 |
+
# Auto-detect mode
|
| 897 |
+
if mode == "auto":
|
| 898 |
+
# Check if running on HF Spaces
|
| 899 |
+
if os.environ.get("SPACE_ID"):
|
| 900 |
+
mode = "hybrid" if token else "local"
|
| 901 |
+
else:
|
| 902 |
+
mode = "local"
|
| 903 |
+
|
| 904 |
+
# Get token from env if not provided
|
| 905 |
+
if not token:
|
| 906 |
+
token = os.environ.get("HF_TOKEN", "") or os.environ.get("HF_AUTH_TOKEN", "")
|
| 907 |
+
|
| 908 |
+
config = HubConfig(
|
| 909 |
+
token=token,
|
| 910 |
+
repo_id=repo_id,
|
| 911 |
+
dataset_repo_id=dataset_repo_id,
|
| 912 |
+
mode=mode,
|
| 913 |
+
**kwargs
|
| 914 |
+
)
|
| 915 |
+
|
| 916 |
+
_global_storage = HubStorage(config=config)
|
| 917 |
+
_global_storage.initialize()
|
| 918 |
+
|
| 919 |
+
return _global_storage
|
| 920 |
+
|
| 921 |
+
|
| 922 |
+
def is_hub_available() -> bool:
|
| 923 |
+
"""Check if Hub storage is available and configured."""
|
| 924 |
+
if _global_storage is None:
|
| 925 |
+
return False
|
| 926 |
+
return _global_storage.config.is_hub_enabled()
|
| 927 |
+
|
| 928 |
+
|
| 929 |
+
def get_storage_mode() -> str:
|
| 930 |
+
"""Get current storage mode."""
|
| 931 |
+
if _global_storage is None:
|
| 932 |
+
return "not_initialized"
|
| 933 |
+
return _global_storage.config.mode
|
|
@@ -1662,3 +1662,360 @@ def handle_push_github(
|
|
| 1662 |
timeout=timeout_int,
|
| 1663 |
)
|
| 1664 |
yield json.dumps(result, default=str)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1662 |
timeout=timeout_int,
|
| 1663 |
)
|
| 1664 |
yield json.dumps(result, default=str)
|
| 1665 |
+
|
| 1666 |
+
|
| 1667 |
+
# ═══════════════════════════════════════════════════════════════════════
|
| 1668 |
+
# HUB STORAGE API ENDPOINTS
|
| 1669 |
+
# ═══════════════════════════════════════════════════════════════════════
|
| 1670 |
+
|
| 1671 |
+
@app.api(name="hub_status")
|
| 1672 |
+
def handle_hub_status() -> str:
|
| 1673 |
+
"""Get current Hub storage status and configuration.
|
| 1674 |
+
|
| 1675 |
+
Returns:
|
| 1676 |
+
JSON dict with: available, configured, mode, hub_enabled,
|
| 1677 |
+
dataset_enabled, local_files, local_size_mb, etc.
|
| 1678 |
+
"""
|
| 1679 |
+
from code.tools.fs import get_hub_status
|
| 1680 |
+
|
| 1681 |
+
result = get_hub_status()
|
| 1682 |
+
yield json.dumps(result, default=str)
|
| 1683 |
+
|
| 1684 |
+
|
| 1685 |
+
@app.api(name="hub_init")
|
| 1686 |
+
def handle_hub_init(
|
| 1687 |
+
token: str = "",
|
| 1688 |
+
repo_id: str = "",
|
| 1689 |
+
dataset_repo_id: str = "",
|
| 1690 |
+
mode: str = "auto",
|
| 1691 |
+
) -> str:
|
| 1692 |
+
"""Initialize/configure Hub storage.
|
| 1693 |
+
|
| 1694 |
+
This sets up the connection to Hugging Face Hub for cloud storage.
|
| 1695 |
+
|
| 1696 |
+
Args:
|
| 1697 |
+
token: HF API token (or use HF_TOKEN env var)
|
| 1698 |
+
repo_id: Main repository ID (e.g., "user/repo")
|
| 1699 |
+
dataset_repo_id: Dataset repo for workspace persistence
|
| 1700 |
+
mode: "local", "hub", "hybrid", or "auto"
|
| 1701 |
+
|
| 1702 |
+
Returns:
|
| 1703 |
+
JSON dict with initialization status and config.
|
| 1704 |
+
"""
|
| 1705 |
+
from code.hub_storage import init_hub_storage
|
| 1706 |
+
|
| 1707 |
+
try:
|
| 1708 |
+
storage = init_hub_storage(
|
| 1709 |
+
token=token,
|
| 1710 |
+
repo_id=repo_id,
|
| 1711 |
+
dataset_repo_id=dataset_repo_id,
|
| 1712 |
+
mode=mode,
|
| 1713 |
+
)
|
| 1714 |
+
|
| 1715 |
+
result = {
|
| 1716 |
+
"success": True,
|
| 1717 |
+
"message": "Hub storage initialized",
|
| 1718 |
+
"config": {
|
| 1719 |
+
"mode": storage.config.mode,
|
| 1720 |
+
"repo_id": storage.config.repo_id,
|
| 1721 |
+
"dataset_repo_id": storage.config.dataset_repo_id,
|
| 1722 |
+
"hub_enabled": storage.config.is_hub_enabled(),
|
| 1723 |
+
"dataset_enabled": storage.config.is_dataset_storage_enabled(),
|
| 1724 |
+
},
|
| 1725 |
+
"stats": storage.get_storage_stats(),
|
| 1726 |
+
}
|
| 1727 |
+
except Exception as e:
|
| 1728 |
+
result = {
|
| 1729 |
+
"success": False,
|
| 1730 |
+
"error": str(e),
|
| 1731 |
+
}
|
| 1732 |
+
|
| 1733 |
+
yield json.dumps(result, default=str)
|
| 1734 |
+
|
| 1735 |
+
|
| 1736 |
+
@app.api(name="hub_sync_push", concurrency_limit=1)
|
| 1737 |
+
def handle_hub_sync_push(path: str = "") -> str:
|
| 1738 |
+
"""Sync local workspace files to HF Hub.
|
| 1739 |
+
|
| 1740 |
+
Args:
|
| 1741 |
+
path: Optional specific file/directory to sync
|
| 1742 |
+
|
| 1743 |
+
Returns:
|
| 1744 |
+
JSON dict with sync results.
|
| 1745 |
+
"""
|
| 1746 |
+
from code.tools.fs import sync_to_hub
|
| 1747 |
+
|
| 1748 |
+
try:
|
| 1749 |
+
result = sync_to_hub(path=path if path else None)
|
| 1750 |
+
yield json.dumps(result, default=str)
|
| 1751 |
+
except Exception as e:
|
| 1752 |
+
yield json.dumps({"success": False, "error": str(e)}, default=str)
|
| 1753 |
+
|
| 1754 |
+
|
| 1755 |
+
@app.api(name="hub_sync_pull", concurrency_limit=1)
|
| 1756 |
+
def handle_hub_sync_pull(path: str = "") -> str:
|
| 1757 |
+
"""Sync files from HF Hub to local workspace.
|
| 1758 |
+
|
| 1759 |
+
Args:
|
| 1760 |
+
path: Optional specific file to download
|
| 1761 |
+
|
| 1762 |
+
Returns:
|
| 1763 |
+
JSON dict with sync results.
|
| 1764 |
+
"""
|
| 1765 |
+
from code.tools.fs import sync_from_hub
|
| 1766 |
+
|
| 1767 |
+
try:
|
| 1768 |
+
result = sync_from_hub(path=path if path else None)
|
| 1769 |
+
yield json.dumps(result, default=str)
|
| 1770 |
+
except Exception as e:
|
| 1771 |
+
yield json.dumps({"success": False, "error": str(e)}, default=str)
|
| 1772 |
+
|
| 1773 |
+
|
| 1774 |
+
@app.api(name="hub_full_sync", concurrency_limit=1)
|
| 1775 |
+
def handle_hub_full_sync() -> str:
|
| 1776 |
+
"""Perform bidirectional sync between local and Hub.
|
| 1777 |
+
|
| 1778 |
+
Merges changes from both sides.
|
| 1779 |
+
|
| 1780 |
+
Returns:
|
| 1781 |
+
JSON dict with both push and pull results.
|
| 1782 |
+
"""
|
| 1783 |
+
from code.hub_storage import get_hub_storage, is_hub_available
|
| 1784 |
+
|
| 1785 |
+
if not is_hub_available():
|
| 1786 |
+
yield json.dumps({
|
| 1787 |
+
"success": False,
|
| 1788 |
+
"error": "Hub storage not configured"
|
| 1789 |
+
}, default=str)
|
| 1790 |
+
return
|
| 1791 |
+
|
| 1792 |
+
try:
|
| 1793 |
+
hub = get_hub_storage()
|
| 1794 |
+
result = hub.full_sync()
|
| 1795 |
+
yield json.dumps(result, default=str)
|
| 1796 |
+
except Exception as e:
|
| 1797 |
+
yield json.dumps({"success": False, "error": str(e)}, default=str)
|
| 1798 |
+
|
| 1799 |
+
|
| 1800 |
+
@app.api(name="hub_list_files")
|
| 1801 |
+
def handle_hub_list_files(path: str = ".", recursive: bool = True) -> str:
|
| 1802 |
+
"""List files in Hub storage.
|
| 1803 |
+
|
| 1804 |
+
Args:
|
| 1805 |
+
path: Directory path to list
|
| 1806 |
+
recursive: Whether to list recursively
|
| 1807 |
+
|
| 1808 |
+
Returns:
|
| 1809 |
+
JSON dict with file listing.
|
| 1810 |
+
"""
|
| 1811 |
+
from code.hub_storage import get_hub_storage, is_hub_available
|
| 1812 |
+
|
| 1813 |
+
if not is_hub_available():
|
| 1814 |
+
yield json.dumps({
|
| 1815 |
+
"success": False,
|
| 1816 |
+
"error": "Hub storage not configured",
|
| 1817 |
+
"files": []
|
| 1818 |
+
}, default_str=str)
|
| 1819 |
+
return
|
| 1820 |
+
|
| 1821 |
+
try:
|
| 1822 |
+
hub = get_hub_storage()
|
| 1823 |
+
files = hub.list_files(path=path, recursive=recursive)
|
| 1824 |
+
yield json.dumps({
|
| 1825 |
+
"success": True,
|
| 1826 |
+
"files": files,
|
| 1827 |
+
"count": len(files),
|
| 1828 |
+
}, default=str)
|
| 1829 |
+
except Exception as e:
|
| 1830 |
+
yield json.dumps({"success": False, "error": str(e), "files": []}, default=str)
|
| 1831 |
+
|
| 1832 |
+
|
| 1833 |
+
@app.api(name="hub_delete_file")
|
| 1834 |
+
def handle_hub_delete_file(path: str) -> str:
|
| 1835 |
+
"""Delete a file from Hub storage.
|
| 1836 |
+
|
| 1837 |
+
Args:
|
| 1838 |
+
path: File path to delete
|
| 1839 |
+
|
| 1840 |
+
Returns:
|
| 1841 |
+
JSON dict with operation result.
|
| 1842 |
+
"""
|
| 1843 |
+
from code.tools.fs import delete_from_hub
|
| 1844 |
+
|
| 1845 |
+
try:
|
| 1846 |
+
result = delete_from_hub(path)
|
| 1847 |
+
yield json.dumps(result, default=str)
|
| 1848 |
+
except Exception as e:
|
| 1849 |
+
yield json.dumps({"success": False, "error": str(e)}, default=str)
|
| 1850 |
+
|
| 1851 |
+
|
| 1852 |
+
@app.api(name="hub_read_file")
|
| 1853 |
+
def handle_hub_read_file(path: str) -> str:
|
| 1854 |
+
"""Read a file specifically from Hub storage.
|
| 1855 |
+
|
| 1856 |
+
Args:
|
| 1857 |
+
path: File path to read
|
| 1858 |
+
|
| 1859 |
+
Returns:
|
| 1860 |
+
JSON dict with file content.
|
| 1861 |
+
"""
|
| 1862 |
+
from code.hub_storage import get_hub_storage, is_hub_available
|
| 1863 |
+
|
| 1864 |
+
if not is_hub_available():
|
| 1865 |
+
yield json.dumps({
|
| 1866 |
+
"success": False,
|
| 1867 |
+
"error": "Hub storage not configured"
|
| 1868 |
+
}, default_str=str)
|
| 1869 |
+
return
|
| 1870 |
+
|
| 1871 |
+
try:
|
| 1872 |
+
hub = get_hub_storage()
|
| 1873 |
+
content = hub.read_file(path)
|
| 1874 |
+
yield json.dumps({
|
| 1875 |
+
"success": True,
|
| 1876 |
+
"path": path,
|
| 1877 |
+
"content": content,
|
| 1878 |
+
"size": len(content),
|
| 1879 |
+
}, default_str=str)
|
| 1880 |
+
except Exception as e:
|
| 1881 |
+
yield json.dumps({"success": False, "error": str(e)}, default_str=str)
|
| 1882 |
+
|
| 1883 |
+
|
| 1884 |
+
@app.api(name="hub_write_file")
|
| 1885 |
+
def handle_hub_write_file(path: str, content: str, commit_message: str = "") -> str:
|
| 1886 |
+
"""Write a file directly to Hub storage.
|
| 1887 |
+
|
| 1888 |
+
This allows the AI to write code that persists on HF Hub.
|
| 1889 |
+
|
| 1890 |
+
Args:
|
| 1891 |
+
path: Destination file path (e.g., "src/app.py")
|
| 1892 |
+
content: File content
|
| 1893 |
+
commit_message: Optional commit message
|
| 1894 |
+
|
| 1895 |
+
Returns:
|
| 1896 |
+
JSON dict with operation result and metadata.
|
| 1897 |
+
"""
|
| 1898 |
+
from code.hub_storage import get_hub_storage, is_hub_available
|
| 1899 |
+
|
| 1900 |
+
if not is_hub_available():
|
| 1901 |
+
yield json.dumps({
|
| 1902 |
+
"success": False,
|
| 1903 |
+
"error": "Hub storage not configured. Initialize with /hub_init first."
|
| 1904 |
+
}, default_str=str)
|
| 1905 |
+
return
|
| 1906 |
+
|
| 1907 |
+
try:
|
| 1908 |
+
hub = get_hub_storage()
|
| 1909 |
+
|
| 1910 |
+
# Use a descriptive commit message if not provided
|
| 1911 |
+
if not commit_message:
|
| 1912 |
+
commit_message = f"AI generated: {path}"
|
| 1913 |
+
|
| 1914 |
+
metadata = hub.write_file(
|
| 1915 |
+
path=path,
|
| 1916 |
+
content=content,
|
| 1917 |
+
commit_message=commit_message,
|
| 1918 |
+
sync_to_hub=True # Force immediate upload
|
| 1919 |
+
)
|
| 1920 |
+
|
| 1921 |
+
yield json.dumps({
|
| 1922 |
+
"success": True,
|
| 1923 |
+
"path": path,
|
| 1924 |
+
"size": metadata.size,
|
| 1925 |
+
"is_remote": metadata.is_remote,
|
| 1926 |
+
"is_local": metadata.is_local,
|
| 1927 |
+
"message": f"File written to {'Hub' if metadata.is_remote else 'local'} storage",
|
| 1928 |
+
}, default_str=str)
|
| 1929 |
+
|
| 1930 |
+
except Exception as e:
|
| 1931 |
+
yield json.dumps({"success": False, "error": str(e)}, default_str=str)
|
| 1932 |
+
|
| 1933 |
+
|
| 1934 |
+
@app.api(name="hub_create_repo")
|
| 1935 |
+
def handle_hub_create_repo(
|
| 1936 |
+
repo_id: str,
|
| 1937 |
+
repo_type: str = "dataset",
|
| 1938 |
+
private: bool = True,
|
| 1939 |
+
token: str = "",
|
| 1940 |
+
) -> str:
|
| 1941 |
+
"""Create a new repository on HF Hub for storage.
|
| 1942 |
+
|
| 1943 |
+
Args:
|
| 1944 |
+
repo_id: Repository ID (e.g., "username/my-workspace")
|
| 1945 |
+
repo_type: "space", "model", or "dataset"
|
| 1946 |
+
private: Whether to make it private
|
| 1947 |
+
token: HF API token
|
| 1948 |
+
|
| 1949 |
+
Returns:
|
| 1950 |
+
JSON dict with creation result.
|
| 1951 |
+
"""
|
| 1952 |
+
from huggingface_hub import HfApi
|
| 1953 |
+
|
| 1954 |
+
try:
|
| 1955 |
+
api = HfApi(token=token) if token else HfApi()
|
| 1956 |
+
|
| 1957 |
+
url = api.create_repo(
|
| 1958 |
+
repo_id=repo_id,
|
| 1959 |
+
repo_type=repo_type,
|
| 1960 |
+
private=private,
|
| 1961 |
+
exist_ok=True,
|
| 1962 |
+
)
|
| 1963 |
+
|
| 1964 |
+
yield json.dumps({
|
| 1965 |
+
"success": True,
|
| 1966 |
+
"repo_id": repo_id,
|
| 1967 |
+
"repo_type": repo_type,
|
| 1968 |
+
"private": private,
|
| 1969 |
+
"url": url,
|
| 1970 |
+
"message": f"Repository {repo_id} created/verified",
|
| 1971 |
+
}, default_str=str)
|
| 1972 |
+
|
| 1973 |
+
except Exception as e:
|
| 1974 |
+
yield json.dumps({
|
| 1975 |
+
"success": False,
|
| 1976 |
+
"error": str(e),
|
| 1977 |
+
}, default_str=str)
|
| 1978 |
+
|
| 1979 |
+
|
| 1980 |
+
@app.api(name="hub_stats")
|
| 1981 |
+
def handle_hub_stats() -> str:
|
| 1982 |
+
"""Get detailed statistics about current storage usage.
|
| 1983 |
+
|
| 1984 |
+
Returns:
|
| 1985 |
+
JSON dict with storage statistics for both local and Hub.
|
| 1986 |
+
"""
|
| 1987 |
+
from code.hub_storage import get_hub_storage, is_hub_available
|
| 1988 |
+
from code.tools.fs import snapshot_workspace
|
| 1989 |
+
|
| 1990 |
+
stats = {
|
| 1991 |
+
"timestamp": __import__("time").time(),
|
| 1992 |
+
"local": {},
|
| 1993 |
+
"hub": None,
|
| 1994 |
+
}
|
| 1995 |
+
|
| 1996 |
+
# Local stats
|
| 1997 |
+
try:
|
| 1998 |
+
workspace_snapshot = snapshot_workspace()
|
| 1999 |
+
total_size = sum(len(c.encode('utf-8')) for c in workspace_snapshot.values())
|
| 2000 |
+
stats["local"] = {
|
| 2001 |
+
"file_count": len(workspace_snapshot),
|
| 2002 |
+
"total_size_bytes": total_size,
|
| 2003 |
+
"total_size_mb": round(total_size / (1024 * 1024), 2),
|
| 2004 |
+
}
|
| 2005 |
+
except Exception as e:
|
| 2006 |
+
stats["local"]["error"] = str(e)
|
| 2007 |
+
|
| 2008 |
+
# Hub stats
|
| 2009 |
+
if is_hub_available():
|
| 2010 |
+
try:
|
| 2011 |
+
hub = get_hub_storage()
|
| 2012 |
+
stats["hub"] = hub.get_storage_stats()
|
| 2013 |
+
|
| 2014 |
+
# Get remote file count
|
| 2015 |
+
hub_files = hub.list_files(use_cache=False)
|
| 2016 |
+
stats["hub"]["remote_file_count"] = len(hub_files)
|
| 2017 |
+
|
| 2018 |
+
except Exception as e:
|
| 2019 |
+
stats["hub"] = {"error": str(e)}
|
| 2020 |
+
|
| 2021 |
+
yield json.dumps(stats, default_str=str)
|
|
@@ -1,5 +1,10 @@
|
|
| 1 |
"""File system tools: read, write, edit, list, glob, grep.
|
| 2 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3 |
All tools are sandboxed to a configurable workspace root (default: ./workspace).
|
| 4 |
They return JSON-serializable dicts so they can be exposed via the API.
|
| 5 |
"""
|
|
@@ -7,11 +12,14 @@ They return JSON-serializable dicts so they can be exposed via the API.
|
|
| 7 |
from __future__ import annotations
|
| 8 |
|
| 9 |
import fnmatch
|
|
|
|
| 10 |
import os
|
| 11 |
import re
|
| 12 |
import shutil
|
| 13 |
from pathlib import Path
|
| 14 |
-
from typing import Any
|
|
|
|
|
|
|
| 15 |
|
| 16 |
# ─── Workspace sandbox ──────────────────────────────────────────────────
|
| 17 |
|
|
@@ -21,6 +29,9 @@ _DEFAULT_WORKSPACE = os.environ.get(
|
|
| 21 |
os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "workspace")),
|
| 22 |
)
|
| 23 |
|
|
|
|
|
|
|
|
|
|
| 24 |
|
| 25 |
def get_workspace_root() -> str:
|
| 26 |
"""Return the absolute path of the agent's workspace root."""
|
|
@@ -49,11 +60,30 @@ def _resolve_safe(path: str) -> str:
|
|
| 49 |
return full
|
| 50 |
|
| 51 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 52 |
# ─── read_file ──────────────────────────────────────────────────────────
|
| 53 |
|
| 54 |
def read_file(path: str, offset: int = 0, limit: int | None = None) -> dict[str, Any]:
|
| 55 |
"""Read a text file from the workspace.
|
| 56 |
|
|
|
|
|
|
|
| 57 |
Args:
|
| 58 |
path: Relative path inside the workspace, or absolute within it.
|
| 59 |
offset: 1-indexed line to start reading from.
|
|
@@ -64,32 +94,62 @@ def read_file(path: str, offset: int = 0, limit: int | None = None) -> dict[str,
|
|
| 64 |
"""
|
| 65 |
try:
|
| 66 |
full = _resolve_safe(path)
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
if os.path.
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
f"{start + i + 1:6}\t{line}" for i, line in enumerate(selected)
|
| 83 |
-
)
|
| 84 |
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 93 |
except Exception as exc:
|
| 94 |
return {"success": False, "error": str(exc)}
|
| 95 |
|
|
@@ -97,17 +157,52 @@ def read_file(path: str, offset: int = 0, limit: int | None = None) -> dict[str,
|
|
| 97 |
# ─── write_file ─────────────────────────────────────────────────────────
|
| 98 |
|
| 99 |
def write_file(path: str, content: str) -> dict[str, Any]:
|
| 100 |
-
"""Write content to a file, creating parent directories as needed.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 101 |
try:
|
| 102 |
full = _resolve_safe(path)
|
| 103 |
os.makedirs(os.path.dirname(full), exist_ok=True)
|
|
|
|
|
|
|
| 104 |
with open(full, "w", encoding="utf-8") as f:
|
| 105 |
f.write(content)
|
| 106 |
-
|
|
|
|
| 107 |
"success": True,
|
| 108 |
"path": path,
|
| 109 |
"bytes_written": len(content.encode("utf-8")),
|
|
|
|
| 110 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 111 |
except Exception as exc:
|
| 112 |
return {"success": False, "error": str(exc)}
|
| 113 |
|
|
@@ -120,15 +215,23 @@ def edit_file(
|
|
| 120 |
new_str: str,
|
| 121 |
replace_all: bool = False,
|
| 122 |
) -> dict[str, Any]:
|
| 123 |
-
"""Replace occurrences of old_str with new_str in a file.
|
|
|
|
|
|
|
|
|
|
| 124 |
try:
|
| 125 |
full = _resolve_safe(path)
|
| 126 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 127 |
return {"success": False, "error": f"File not found: {path}"}
|
| 128 |
|
| 129 |
-
with open(full, "r", encoding="utf-8") as f:
|
| 130 |
-
content = f.read()
|
| 131 |
-
|
| 132 |
if old_str not in content:
|
| 133 |
return {
|
| 134 |
"success": False,
|
|
@@ -152,28 +255,49 @@ def edit_file(
|
|
| 152 |
old_str, new_str, 1
|
| 153 |
)
|
| 154 |
|
|
|
|
|
|
|
| 155 |
with open(full, "w", encoding="utf-8") as f:
|
| 156 |
f.write(new_content)
|
| 157 |
|
| 158 |
-
|
| 159 |
"success": True,
|
| 160 |
"path": path,
|
| 161 |
"replacements": count,
|
| 162 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 163 |
except Exception as exc:
|
| 164 |
return {"success": False, "error": str(exc)}
|
| 165 |
|
| 166 |
|
| 167 |
def multi_edit(path: str, edits: list[dict[str, Any]]) -> dict[str, Any]:
|
| 168 |
-
"""Apply multiple edits to a file atomically (all-or-nothing).
|
|
|
|
|
|
|
|
|
|
| 169 |
try:
|
| 170 |
full = _resolve_safe(path)
|
| 171 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 172 |
return {"success": False, "error": f"File not found: {path}"}
|
| 173 |
|
| 174 |
-
with open(full, "r", encoding="utf-8") as f:
|
| 175 |
-
content = f.read()
|
| 176 |
-
|
| 177 |
applied = 0
|
| 178 |
for edit in edits:
|
| 179 |
old_str = edit.get("old_str", "")
|
|
@@ -196,10 +320,23 @@ def multi_edit(path: str, edits: list[dict[str, Any]]) -> dict[str, Any]:
|
|
| 196 |
)
|
| 197 |
applied += 1
|
| 198 |
|
|
|
|
|
|
|
| 199 |
with open(full, "w", encoding="utf-8") as f:
|
| 200 |
f.write(content)
|
| 201 |
|
| 202 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 203 |
except Exception as exc:
|
| 204 |
return {"success": False, "error": str(exc)}
|
| 205 |
|
|
@@ -207,30 +344,56 @@ def multi_edit(path: str, edits: list[dict[str, Any]]) -> dict[str, Any]:
|
|
| 207 |
# ─── list_dir ───────────────────────────────────────────────────────────
|
| 208 |
|
| 209 |
def list_dir(path: str = ".") -> dict[str, Any]:
|
| 210 |
-
"""List directory contents.
|
|
|
|
|
|
|
|
|
|
| 211 |
try:
|
| 212 |
full = _resolve_safe(path)
|
| 213 |
-
|
| 214 |
-
return {"success": False, "error": f"Path not found: {path}"}
|
| 215 |
-
if not os.path.isdir(full):
|
| 216 |
-
return {"success": False, "error": f"Not a directory: {path}"}
|
| 217 |
-
|
| 218 |
entries = []
|
| 219 |
-
|
| 220 |
-
|
| 221 |
-
|
| 222 |
-
|
| 223 |
-
|
| 224 |
-
|
| 225 |
-
|
| 226 |
-
|
| 227 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 228 |
|
| 229 |
return {
|
| 230 |
"success": True,
|
| 231 |
"path": path,
|
| 232 |
-
"entries": entries,
|
| 233 |
}
|
|
|
|
| 234 |
except Exception as exc:
|
| 235 |
return {"success": False, "error": str(exc)}
|
| 236 |
|
|
@@ -238,18 +401,43 @@ def list_dir(path: str = ".") -> dict[str, Any]:
|
|
| 238 |
# ─── glob ───────────────────────────────────────────────────────────────
|
| 239 |
|
| 240 |
def glob_paths(pattern: str, path: str = ".") -> dict[str, Any]:
|
| 241 |
-
"""Glob file paths matching a pattern, recursively.
|
|
|
|
|
|
|
|
|
|
| 242 |
try:
|
| 243 |
full = _resolve_safe(path)
|
| 244 |
-
matches: list[str] = []
|
| 245 |
-
|
| 246 |
-
|
| 247 |
-
|
| 248 |
-
|
| 249 |
-
|
| 250 |
-
|
| 251 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 252 |
return {"success": True, "pattern": pattern, "matches": matches}
|
|
|
|
| 253 |
except Exception as exc:
|
| 254 |
return {"success": False, "error": str(exc)}
|
| 255 |
|
|
@@ -263,26 +451,61 @@ def grep_search(
|
|
| 263 |
ignore_case: bool = False,
|
| 264 |
max_results: int = 100,
|
| 265 |
) -> dict[str, Any]:
|
| 266 |
-
"""Search file contents with a regex pattern.
|
|
|
|
|
|
|
|
|
|
| 267 |
try:
|
| 268 |
-
full = _resolve_safe(path)
|
| 269 |
flags = re.IGNORECASE if ignore_case else 0
|
| 270 |
regex = re.compile(pattern, flags)
|
| 271 |
|
| 272 |
matches: list[dict[str, Any]] = []
|
| 273 |
-
|
| 274 |
-
|
| 275 |
-
|
| 276 |
-
|
| 277 |
-
|
| 278 |
-
|
| 279 |
-
|
| 280 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 281 |
if regex.search(line):
|
| 282 |
matches.append({
|
| 283 |
-
"file":
|
| 284 |
"line": lineno,
|
| 285 |
"text": line.rstrip()[:500],
|
|
|
|
| 286 |
})
|
| 287 |
if len(matches) >= max_results:
|
| 288 |
return {
|
|
@@ -291,8 +514,10 @@ def grep_search(
|
|
| 291 |
"matches": matches,
|
| 292 |
"truncated": True,
|
| 293 |
}
|
| 294 |
-
|
| 295 |
-
|
|
|
|
|
|
|
| 296 |
|
| 297 |
return {
|
| 298 |
"success": True,
|
|
@@ -300,6 +525,7 @@ def grep_search(
|
|
| 300 |
"matches": matches,
|
| 301 |
"truncated": False,
|
| 302 |
}
|
|
|
|
| 303 |
except re.error as exc:
|
| 304 |
return {"success": False, "error": f"Invalid regex: {exc}"}
|
| 305 |
except Exception as exc:
|
|
@@ -309,7 +535,10 @@ def grep_search(
|
|
| 309 |
# ─── Workspace management ───────────────────────────────────────────────
|
| 310 |
|
| 311 |
def list_workspace_tree(max_depth: int = 3) -> dict[str, Any]:
|
| 312 |
-
"""Return a tree view of the workspace.
|
|
|
|
|
|
|
|
|
|
| 313 |
try:
|
| 314 |
root = get_workspace_root()
|
| 315 |
|
|
@@ -333,13 +562,25 @@ def list_workspace_tree(max_depth: int = 3) -> dict[str, Any]:
|
|
| 333 |
return {"name": os.path.basename(path), "type": "dir", "children": entries}
|
| 334 |
|
| 335 |
tree = _walk(root, 0)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 336 |
return {"success": True, "tree": tree}
|
|
|
|
| 337 |
except Exception as exc:
|
| 338 |
return {"success": False, "error": str(exc)}
|
| 339 |
|
| 340 |
|
| 341 |
def reset_workspace() -> dict[str, Any]:
|
| 342 |
-
"""Clear all files in the workspace (used by /new command).
|
|
|
|
|
|
|
|
|
|
| 343 |
try:
|
| 344 |
root = get_workspace_root()
|
| 345 |
if os.path.exists(root):
|
|
@@ -357,12 +598,14 @@ def reset_workspace() -> dict[str, Any]:
|
|
| 357 |
def snapshot_workspace() -> dict[str, str]:
|
| 358 |
"""Return a dict of {relative_path: content} for all text files in the workspace.
|
| 359 |
|
|
|
|
| 360 |
Used to package workspace files for ZIP/HF deploy.
|
| 361 |
"""
|
| 362 |
root = get_workspace_root()
|
| 363 |
files: dict[str, str] = {}
|
|
|
|
|
|
|
| 364 |
for dirpath, _dirs, fnames in os.walk(root):
|
| 365 |
-
# Skip hidden dirs and node_modules / __pycache__
|
| 366 |
parts = os.path.relpath(dirpath, root).split(os.sep)
|
| 367 |
if any(p.startswith(".") or p in {"node_modules", "__pycache__", ".venv", "venv"} for p in parts):
|
| 368 |
continue
|
|
@@ -375,4 +618,105 @@ def snapshot_workspace() -> dict[str, str]:
|
|
| 375 |
files[os.path.relpath(full, root)] = f.read()
|
| 376 |
except (UnicodeDecodeError, PermissionError):
|
| 377 |
continue
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 378 |
return files
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
"""File system tools: read, write, edit, list, glob, grep.
|
| 2 |
|
| 3 |
+
Enhanced with Hugging Face Hub cloud storage integration:
|
| 4 |
+
- All file operations work seamlessly with both local and cloud storage
|
| 5 |
+
- Automatic sync to HF Hub when configured
|
| 6 |
+
- Transparent fallback between local and remote
|
| 7 |
+
|
| 8 |
All tools are sandboxed to a configurable workspace root (default: ./workspace).
|
| 9 |
They return JSON-serializable dicts so they can be exposed via the API.
|
| 10 |
"""
|
|
|
|
| 12 |
from __future__ import annotations
|
| 13 |
|
| 14 |
import fnmatch
|
| 15 |
+
import logging
|
| 16 |
import os
|
| 17 |
import re
|
| 18 |
import shutil
|
| 19 |
from pathlib import Path
|
| 20 |
+
from typing import Any, Optional
|
| 21 |
+
|
| 22 |
+
logger = logging.getLogger(__name__)
|
| 23 |
|
| 24 |
# ─── Workspace sandbox ──────────────────────────────────────────────────
|
| 25 |
|
|
|
|
| 29 |
os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "workspace")),
|
| 30 |
)
|
| 31 |
|
| 32 |
+
# Hub storage instance (lazy initialized)
|
| 33 |
+
_hub_storage = None
|
| 34 |
+
|
| 35 |
|
| 36 |
def get_workspace_root() -> str:
|
| 37 |
"""Return the absolute path of the agent's workspace root."""
|
|
|
|
| 60 |
return full
|
| 61 |
|
| 62 |
|
| 63 |
+
def _get_hub_storage():
|
| 64 |
+
"""Get the HubStorage instance if available."""
|
| 65 |
+
global _hub_storage
|
| 66 |
+
if _hub_storage is None:
|
| 67 |
+
try:
|
| 68 |
+
from code.hub_storage import get_hub_storage, is_hub_available
|
| 69 |
+
if is_hub_available():
|
| 70 |
+
_hub_storage = get_hub_storage()
|
| 71 |
+
logger.debug("Hub storage integration active")
|
| 72 |
+
else:
|
| 73 |
+
logger.debug("Hub storage not configured")
|
| 74 |
+
except ImportError:
|
| 75 |
+
logger.debug("huggingface_hub not installed")
|
| 76 |
+
_hub_storage = None
|
| 77 |
+
return _hub_storage
|
| 78 |
+
|
| 79 |
+
|
| 80 |
# ─── read_file ──────────────────────────────────────────────────────────
|
| 81 |
|
| 82 |
def read_file(path: str, offset: int = 0, limit: int | None = None) -> dict[str, Any]:
|
| 83 |
"""Read a text file from the workspace.
|
| 84 |
|
| 85 |
+
Enhanced: Falls back to Hub storage if file not found locally.
|
| 86 |
+
|
| 87 |
Args:
|
| 88 |
path: Relative path inside the workspace, or absolute within it.
|
| 89 |
offset: 1-indexed line to start reading from.
|
|
|
|
| 94 |
"""
|
| 95 |
try:
|
| 96 |
full = _resolve_safe(path)
|
| 97 |
+
|
| 98 |
+
# Try local first
|
| 99 |
+
if os.path.exists(full) and os.path.isfile(full):
|
| 100 |
+
with open(full, "r", encoding="utf-8", errors="replace") as f:
|
| 101 |
+
lines = f.readlines()
|
| 102 |
+
|
| 103 |
+
total = len(lines)
|
| 104 |
+
start = max(0, (offset - 1) if offset > 0 else 0)
|
| 105 |
+
end = (start + limit) if limit else total
|
| 106 |
+
selected = lines[start:end]
|
| 107 |
+
|
| 108 |
+
# Re-number for display
|
| 109 |
+
numbered = "".join(
|
| 110 |
+
f"{start + i + 1:6}\t{line}" for i, line in enumerate(selected)
|
| 111 |
+
)
|
|
|
|
|
|
|
| 112 |
|
| 113 |
+
return {
|
| 114 |
+
"success": True,
|
| 115 |
+
"path": path,
|
| 116 |
+
"content": numbered,
|
| 117 |
+
"line_count": total,
|
| 118 |
+
"returned_lines": len(selected),
|
| 119 |
+
"truncated": end < total,
|
| 120 |
+
"source": "local",
|
| 121 |
+
}
|
| 122 |
+
|
| 123 |
+
# Try Hub storage
|
| 124 |
+
hub = _get_hub_storage()
|
| 125 |
+
if hub and hub.file_exists(path):
|
| 126 |
+
try:
|
| 127 |
+
content = hub.read_file(path)
|
| 128 |
+
lines = content.split('\n')
|
| 129 |
+
|
| 130 |
+
total = len(lines)
|
| 131 |
+
start = max(0, (offset - 1) if offset > 0 else 0)
|
| 132 |
+
end = (start + limit) if limit else total
|
| 133 |
+
selected = lines[start:end]
|
| 134 |
+
|
| 135 |
+
numbered = "".join(
|
| 136 |
+
f"{start + i + 1:6}\t{line}" for i, line in enumerate(selected)
|
| 137 |
+
)
|
| 138 |
+
|
| 139 |
+
return {
|
| 140 |
+
"success": True,
|
| 141 |
+
"path": path,
|
| 142 |
+
"content": numbered,
|
| 143 |
+
"line_count": total,
|
| 144 |
+
"returned_lines": len(selected),
|
| 145 |
+
"truncated": end < total,
|
| 146 |
+
"source": "hub",
|
| 147 |
+
}
|
| 148 |
+
except Exception as e:
|
| 149 |
+
logger.warning("Failed to read %s from Hub: %s", path, e)
|
| 150 |
+
|
| 151 |
+
return {"success": False, "error": f"File not found: {path}"}
|
| 152 |
+
|
| 153 |
except Exception as exc:
|
| 154 |
return {"success": False, "error": str(exc)}
|
| 155 |
|
|
|
|
| 157 |
# ─── write_file ─────────────────────────────────────────────────────────
|
| 158 |
|
| 159 |
def write_file(path: str, content: str) -> dict[str, Any]:
|
| 160 |
+
"""Write content to a file, creating parent directories as needed.
|
| 161 |
+
|
| 162 |
+
Enhanced: Automatically syncs to HF Hub when configured.
|
| 163 |
+
|
| 164 |
+
Args:
|
| 165 |
+
path: Relative file path in workspace
|
| 166 |
+
content: File content string
|
| 167 |
+
|
| 168 |
+
Returns:
|
| 169 |
+
dict with success status and metadata
|
| 170 |
+
"""
|
| 171 |
try:
|
| 172 |
full = _resolve_safe(path)
|
| 173 |
os.makedirs(os.path.dirname(full), exist_ok=True)
|
| 174 |
+
|
| 175 |
+
# Write locally first
|
| 176 |
with open(full, "w", encoding="utf-8") as f:
|
| 177 |
f.write(content)
|
| 178 |
+
|
| 179 |
+
result = {
|
| 180 |
"success": True,
|
| 181 |
"path": path,
|
| 182 |
"bytes_written": len(content.encode("utf-8")),
|
| 183 |
+
"source": "local",
|
| 184 |
}
|
| 185 |
+
|
| 186 |
+
# Sync to Hub if configured
|
| 187 |
+
hub = _get_hub_storage()
|
| 188 |
+
if hub:
|
| 189 |
+
try:
|
| 190 |
+
metadata = hub.write_file(
|
| 191 |
+
path=path,
|
| 192 |
+
content=content,
|
| 193 |
+
commit_message=f"AI generated/updated: {path}"
|
| 194 |
+
)
|
| 195 |
+
result["hub_sync"] = True
|
| 196 |
+
result["hub_remote"] = metadata.is_remote
|
| 197 |
+
result["source"] = "synced"
|
| 198 |
+
logger.info("Synced %s to Hub", path)
|
| 199 |
+
except Exception as e:
|
| 200 |
+
logger.warning("Failed to sync %s to Hub: %s", path, e)
|
| 201 |
+
result["hub_sync"] = False
|
| 202 |
+
result["hub_error"] = str(e)
|
| 203 |
+
|
| 204 |
+
return result
|
| 205 |
+
|
| 206 |
except Exception as exc:
|
| 207 |
return {"success": False, "error": str(exc)}
|
| 208 |
|
|
|
|
| 215 |
new_str: str,
|
| 216 |
replace_all: bool = False,
|
| 217 |
) -> dict[str, Any]:
|
| 218 |
+
"""Replace occurrences of old_str with new_str in a file.
|
| 219 |
+
|
| 220 |
+
Enhanced: Syncs changes back to Hub after edit.
|
| 221 |
+
"""
|
| 222 |
try:
|
| 223 |
full = _resolve_safe(path)
|
| 224 |
+
|
| 225 |
+
# Read file (local or Hub)
|
| 226 |
+
hub = _get_hub_storage()
|
| 227 |
+
if os.path.exists(full):
|
| 228 |
+
with open(full, "r", encoding="utf-8") as f:
|
| 229 |
+
content = f.read()
|
| 230 |
+
elif hub and hub.file_exists(path):
|
| 231 |
+
content = hub.read_file(path)
|
| 232 |
+
else:
|
| 233 |
return {"success": False, "error": f"File not found: {path}"}
|
| 234 |
|
|
|
|
|
|
|
|
|
|
| 235 |
if old_str not in content:
|
| 236 |
return {
|
| 237 |
"success": False,
|
|
|
|
| 255 |
old_str, new_str, 1
|
| 256 |
)
|
| 257 |
|
| 258 |
+
# Write locally
|
| 259 |
+
os.makedirs(os.path.dirname(full), exist_ok=True)
|
| 260 |
with open(full, "w", encoding="utf-8") as f:
|
| 261 |
f.write(new_content)
|
| 262 |
|
| 263 |
+
result = {
|
| 264 |
"success": True,
|
| 265 |
"path": path,
|
| 266 |
"replacements": count,
|
| 267 |
}
|
| 268 |
+
|
| 269 |
+
# Sync to Hub
|
| 270 |
+
if hub:
|
| 271 |
+
try:
|
| 272 |
+
hub.write_file(path, new_content, commit_message=f"Edit: {path}")
|
| 273 |
+
result["hub_sync"] = True
|
| 274 |
+
except Exception as e:
|
| 275 |
+
logger.warning("Failed to sync edit to Hub: %s", e)
|
| 276 |
+
|
| 277 |
+
return result
|
| 278 |
+
|
| 279 |
except Exception as exc:
|
| 280 |
return {"success": False, "error": str(exc)}
|
| 281 |
|
| 282 |
|
| 283 |
def multi_edit(path: str, edits: list[dict[str, Any]]) -> dict[str, Any]:
|
| 284 |
+
"""Apply multiple edits to a file atomically (all-or-nothing).
|
| 285 |
+
|
| 286 |
+
Enhanced: Uses batch operation for Hub sync.
|
| 287 |
+
"""
|
| 288 |
try:
|
| 289 |
full = _resolve_safe(path)
|
| 290 |
+
|
| 291 |
+
# Read file
|
| 292 |
+
hub = _get_hub_storage()
|
| 293 |
+
if os.path.exists(full):
|
| 294 |
+
with open(full, "r", encoding="utf-8") as f:
|
| 295 |
+
content = f.read()
|
| 296 |
+
elif hub and hub.file_exists(path):
|
| 297 |
+
content = hub.read_file(path)
|
| 298 |
+
else:
|
| 299 |
return {"success": False, "error": f"File not found: {path}"}
|
| 300 |
|
|
|
|
|
|
|
|
|
|
| 301 |
applied = 0
|
| 302 |
for edit in edits:
|
| 303 |
old_str = edit.get("old_str", "")
|
|
|
|
| 320 |
)
|
| 321 |
applied += 1
|
| 322 |
|
| 323 |
+
# Write locally
|
| 324 |
+
os.makedirs(os.path.dirname(full), exist_ok=True)
|
| 325 |
with open(full, "w", encoding="utf-8") as f:
|
| 326 |
f.write(content)
|
| 327 |
|
| 328 |
+
result = {"success": True, "path": path, "applied": applied}
|
| 329 |
+
|
| 330 |
+
# Sync to Hub
|
| 331 |
+
if hub:
|
| 332 |
+
try:
|
| 333 |
+
hub.write_file(path, content, commit_message=f"Multi-edit: {path}")
|
| 334 |
+
result["hub_sync"] = True
|
| 335 |
+
except Exception as e:
|
| 336 |
+
logger.warning("Failed to sync multi-edit to Hub: %s", e)
|
| 337 |
+
|
| 338 |
+
return result
|
| 339 |
+
|
| 340 |
except Exception as exc:
|
| 341 |
return {"success": False, "error": str(exc)}
|
| 342 |
|
|
|
|
| 344 |
# ─── list_dir ───────────────────────────────────────────────────────────
|
| 345 |
|
| 346 |
def list_dir(path: str = ".") -> dict[str, Any]:
|
| 347 |
+
"""List directory contents.
|
| 348 |
+
|
| 349 |
+
Enhanced: Merges local and Hub files.
|
| 350 |
+
"""
|
| 351 |
try:
|
| 352 |
full = _resolve_safe(path)
|
| 353 |
+
|
|
|
|
|
|
|
|
|
|
|
|
|
| 354 |
entries = []
|
| 355 |
+
seen_names = set()
|
| 356 |
+
|
| 357 |
+
# List local files
|
| 358 |
+
if os.path.exists(full) and os.path.isdir(full):
|
| 359 |
+
for name in sorted(os.listdir(full)):
|
| 360 |
+
entry_path = os.path.join(full, name)
|
| 361 |
+
stat = os.stat(entry_path)
|
| 362 |
+
rel_path = os.path.relpath(entry_path, get_workspace_root())
|
| 363 |
+
entries.append({
|
| 364 |
+
"name": name,
|
| 365 |
+
"type": "dir" if os.path.isdir(entry_path) else "file",
|
| 366 |
+
"size": stat.st_size,
|
| 367 |
+
"path": rel_path.replace(os.sep, "/"),
|
| 368 |
+
"source": "local",
|
| 369 |
+
})
|
| 370 |
+
seen_names.add(name)
|
| 371 |
+
|
| 372 |
+
# Merge with Hub files
|
| 373 |
+
hub = _get_hub_storage()
|
| 374 |
+
if hub:
|
| 375 |
+
try:
|
| 376 |
+
hub_files = hub.list_files(path=path, recursive=False)
|
| 377 |
+
for hf in hub_files:
|
| 378 |
+
hf_name = Path(hf["path"]).name
|
| 379 |
+
if hf_name not in seen_names:
|
| 380 |
+
entries.append({
|
| 381 |
+
"name": hf_name,
|
| 382 |
+
"type": "file",
|
| 383 |
+
"size": hf.get("size", 0),
|
| 384 |
+
"path": hf["path"],
|
| 385 |
+
"source": "hub",
|
| 386 |
+
})
|
| 387 |
+
seen_names.add(hf_name)
|
| 388 |
+
except Exception as e:
|
| 389 |
+
logger.warning("Failed to list Hub files: %s", e)
|
| 390 |
|
| 391 |
return {
|
| 392 |
"success": True,
|
| 393 |
"path": path,
|
| 394 |
+
"entries": sorted(entries, key=lambda x: x["name"]),
|
| 395 |
}
|
| 396 |
+
|
| 397 |
except Exception as exc:
|
| 398 |
return {"success": False, "error": str(exc)}
|
| 399 |
|
|
|
|
| 401 |
# ─── glob ───────────────────────────────────────────────────────────────
|
| 402 |
|
| 403 |
def glob_paths(pattern: str, path: str = ".") -> dict[str, Any]:
|
| 404 |
+
"""Glob file paths matching a pattern, recursively.
|
| 405 |
+
|
| 406 |
+
Enhanced: Searches both local and Hub storage.
|
| 407 |
+
"""
|
| 408 |
try:
|
| 409 |
full = _resolve_safe(path)
|
| 410 |
+
matches: list[dict[str, Any]] = []
|
| 411 |
+
seen_paths = set()
|
| 412 |
+
|
| 413 |
+
# Local glob
|
| 414 |
+
if os.path.exists(full):
|
| 415 |
+
for root_dir, _dirs, files in os.walk(full):
|
| 416 |
+
for fname in files:
|
| 417 |
+
if fnmatch.fnmatch(fname, pattern) or fnmatch.fnmatch(
|
| 418 |
+
os.path.relpath(os.path.join(root_dir, fname), full), pattern
|
| 419 |
+
):
|
| 420 |
+
rel_path = os.path.relpath(os.path.join(root_dir, fname), get_workspace_root()).replace(os.sep, "/")
|
| 421 |
+
if rel_path not in seen_paths:
|
| 422 |
+
matches.append({"path": rel_path, "source": "local"})
|
| 423 |
+
seen_paths.add(rel_path)
|
| 424 |
+
|
| 425 |
+
# Hub glob
|
| 426 |
+
hub = _get_hub_storage()
|
| 427 |
+
if hub:
|
| 428 |
+
try:
|
| 429 |
+
hub_files = hub.list_files(path=path, recursive=True)
|
| 430 |
+
for hf in hub_files:
|
| 431 |
+
if fnmatch.fnmatch(Path(hf["path"]).name, pattern) or fnmatch.fnmatch(hf["path"], pattern):
|
| 432 |
+
if hf["path"] not in seen_paths:
|
| 433 |
+
matches.append({"path": hf["path"], "source": "hub"})
|
| 434 |
+
seen_paths.add(hf["path"])
|
| 435 |
+
except Exception as e:
|
| 436 |
+
logger.warning("Failed to glob Hub files: %s", e)
|
| 437 |
+
|
| 438 |
+
matches.sort(key=lambda x: x["path"])
|
| 439 |
return {"success": True, "pattern": pattern, "matches": matches}
|
| 440 |
+
|
| 441 |
except Exception as exc:
|
| 442 |
return {"success": False, "error": str(exc)}
|
| 443 |
|
|
|
|
| 451 |
ignore_case: bool = False,
|
| 452 |
max_results: int = 100,
|
| 453 |
) -> dict[str, Any]:
|
| 454 |
+
"""Search file contents with a regex pattern.
|
| 455 |
+
|
| 456 |
+
Enhanced: Searches both local and Hub files.
|
| 457 |
+
"""
|
| 458 |
try:
|
|
|
|
| 459 |
flags = re.IGNORECASE if ignore_case else 0
|
| 460 |
regex = re.compile(pattern, flags)
|
| 461 |
|
| 462 |
matches: list[dict[str, Any]] = []
|
| 463 |
+
|
| 464 |
+
# Search local files
|
| 465 |
+
full = _resolve_safe(path)
|
| 466 |
+
if os.path.exists(full):
|
| 467 |
+
for root_dir, _dirs, files in os.walk(full):
|
| 468 |
+
for fname in files:
|
| 469 |
+
if include and not fnmatch.fnmatch(fname, include):
|
| 470 |
+
continue
|
| 471 |
+
fpath = os.path.join(root_dir, fname)
|
| 472 |
+
try:
|
| 473 |
+
with open(fpath, "r", encoding="utf-8", errors="replace") as f:
|
| 474 |
+
for lineno, line in enumerate(f, 1):
|
| 475 |
+
if regex.search(line):
|
| 476 |
+
matches.append({
|
| 477 |
+
"file": os.path.relpath(fpath, get_workspace_root()).replace(os.sep, "/"),
|
| 478 |
+
"line": lineno,
|
| 479 |
+
"text": line.rstrip()[:500],
|
| 480 |
+
"source": "local",
|
| 481 |
+
})
|
| 482 |
+
if len(matches) >= max_results:
|
| 483 |
+
return {
|
| 484 |
+
"success": True,
|
| 485 |
+
"pattern": pattern,
|
| 486 |
+
"matches": matches,
|
| 487 |
+
"truncated": True,
|
| 488 |
+
}
|
| 489 |
+
except (UnicodeDecodeError, PermissionError):
|
| 490 |
+
continue
|
| 491 |
+
|
| 492 |
+
# Search Hub files
|
| 493 |
+
hub = _get_hub_storage()
|
| 494 |
+
if hub and len(matches) < max_results:
|
| 495 |
+
try:
|
| 496 |
+
hub_files = hub.list_files(path=path, recursive=True)
|
| 497 |
+
for hf in hub_files:
|
| 498 |
+
if include and not fnmatch.fnmatch(Path(hf["path"]).name, include):
|
| 499 |
+
continue
|
| 500 |
+
try:
|
| 501 |
+
content = hub.read_file(hf["path"])
|
| 502 |
+
for lineno, line in enumerate(content.split('\n'), 1):
|
| 503 |
if regex.search(line):
|
| 504 |
matches.append({
|
| 505 |
+
"file": hf["path"],
|
| 506 |
"line": lineno,
|
| 507 |
"text": line.rstrip()[:500],
|
| 508 |
+
"source": "hub",
|
| 509 |
})
|
| 510 |
if len(matches) >= max_results:
|
| 511 |
return {
|
|
|
|
| 514 |
"matches": matches,
|
| 515 |
"truncated": True,
|
| 516 |
}
|
| 517 |
+
except Exception:
|
| 518 |
+
continue
|
| 519 |
+
except Exception as e:
|
| 520 |
+
logger.warning("Failed to search Hub files: %s", e)
|
| 521 |
|
| 522 |
return {
|
| 523 |
"success": True,
|
|
|
|
| 525 |
"matches": matches,
|
| 526 |
"truncated": False,
|
| 527 |
}
|
| 528 |
+
|
| 529 |
except re.error as exc:
|
| 530 |
return {"success": False, "error": f"Invalid regex: {exc}"}
|
| 531 |
except Exception as exc:
|
|
|
|
| 535 |
# ─── Workspace management ───────────────────────────────────────────────
|
| 536 |
|
| 537 |
def list_workspace_tree(max_depth: int = 3) -> dict[str, Any]:
|
| 538 |
+
"""Return a tree view of the workspace.
|
| 539 |
+
|
| 540 |
+
Enhanced: Includes Hub files.
|
| 541 |
+
"""
|
| 542 |
try:
|
| 543 |
root = get_workspace_root()
|
| 544 |
|
|
|
|
| 562 |
return {"name": os.path.basename(path), "type": "dir", "children": entries}
|
| 563 |
|
| 564 |
tree = _walk(root, 0)
|
| 565 |
+
|
| 566 |
+
# Add Hub info
|
| 567 |
+
hub = _get_hub_storage()
|
| 568 |
+
if hub:
|
| 569 |
+
stats = hub.get_storage_stats()
|
| 570 |
+
tree["hub_enabled"] = stats["hub_enabled"]
|
| 571 |
+
tree["storage_mode"] = stats["mode"]
|
| 572 |
+
|
| 573 |
return {"success": True, "tree": tree}
|
| 574 |
+
|
| 575 |
except Exception as exc:
|
| 576 |
return {"success": False, "error": str(exc)}
|
| 577 |
|
| 578 |
|
| 579 |
def reset_workspace() -> dict[str, Any]:
|
| 580 |
+
"""Clear all files in the workspace (used by /new command).
|
| 581 |
+
|
| 582 |
+
Note: Does NOT clear Hub storage - use reset_hub_workspace for that.
|
| 583 |
+
"""
|
| 584 |
try:
|
| 585 |
root = get_workspace_root()
|
| 586 |
if os.path.exists(root):
|
|
|
|
| 598 |
def snapshot_workspace() -> dict[str, str]:
|
| 599 |
"""Return a dict of {relative_path: content} for all text files in the workspace.
|
| 600 |
|
| 601 |
+
Enhanced: Includes Hub files in snapshot.
|
| 602 |
Used to package workspace files for ZIP/HF deploy.
|
| 603 |
"""
|
| 604 |
root = get_workspace_root()
|
| 605 |
files: dict[str, str] = {}
|
| 606 |
+
|
| 607 |
+
# Local files
|
| 608 |
for dirpath, _dirs, fnames in os.walk(root):
|
|
|
|
| 609 |
parts = os.path.relpath(dirpath, root).split(os.sep)
|
| 610 |
if any(p.startswith(".") or p in {"node_modules", "__pycache__", ".venv", "venv"} for p in parts):
|
| 611 |
continue
|
|
|
|
| 618 |
files[os.path.relpath(full, root)] = f.read()
|
| 619 |
except (UnicodeDecodeError, PermissionError):
|
| 620 |
continue
|
| 621 |
+
|
| 622 |
+
# Hub files (merge, don't overwrite local)
|
| 623 |
+
hub = _get_hub_storage()
|
| 624 |
+
if hub:
|
| 625 |
+
try:
|
| 626 |
+
hub_files = hub.list_files()
|
| 627 |
+
for hf in hub_files:
|
| 628 |
+
if hf["path"] not in files:
|
| 629 |
+
try:
|
| 630 |
+
content = hub.read_file(hf["path"])
|
| 631 |
+
files[hf["path"]] = content
|
| 632 |
+
except Exception:
|
| 633 |
+
continue
|
| 634 |
+
except Exception as e:
|
| 635 |
+
logger.warning("Failed to snapshot Hub files: %s", e)
|
| 636 |
+
|
| 637 |
return files
|
| 638 |
+
|
| 639 |
+
|
| 640 |
+
# ─── Hub-specific operations ────────────────────────────────────────────
|
| 641 |
+
|
| 642 |
+
def sync_to_hub(path: str | None = None) -> dict[str, Any]:
|
| 643 |
+
"""Manually trigger sync to HF Hub.
|
| 644 |
+
|
| 645 |
+
Args:
|
| 646 |
+
path: Specific file/directory to sync (None = all)
|
| 647 |
+
|
| 648 |
+
Returns:
|
| 649 |
+
Sync result dict
|
| 650 |
+
"""
|
| 651 |
+
hub = _get_hub_storage()
|
| 652 |
+
if not hub:
|
| 653 |
+
return {"success": False, "error": "Hub storage not configured"}
|
| 654 |
+
|
| 655 |
+
try:
|
| 656 |
+
return hub.sync_to_hub(path=path)
|
| 657 |
+
except Exception as e:
|
| 658 |
+
return {"success": False, "error": str(e)}
|
| 659 |
+
|
| 660 |
+
|
| 661 |
+
def sync_from_hub(path: str | None = None) -> dict[str, Any]:
|
| 662 |
+
"""Manually trigger sync from HF Hub.
|
| 663 |
+
|
| 664 |
+
Args:
|
| 665 |
+
path: Specific file to download (None = all)
|
| 666 |
+
|
| 667 |
+
Returns:
|
| 668 |
+
Sync result dict
|
| 669 |
+
"""
|
| 670 |
+
hub = _get_hub_storage()
|
| 671 |
+
if not hub:
|
| 672 |
+
return {"success": False, "error": "Hub storage not configured"}
|
| 673 |
+
|
| 674 |
+
try:
|
| 675 |
+
return hub.sync_from_hub(path=path)
|
| 676 |
+
except Exception as e:
|
| 677 |
+
return {"success": False, "error": str(e)}
|
| 678 |
+
|
| 679 |
+
|
| 680 |
+
def get_hub_status() -> dict[str, Any]:
|
| 681 |
+
"""Get current Hub storage status and configuration."""
|
| 682 |
+
hub = _get_hub_storage()
|
| 683 |
+
if not hub:
|
| 684 |
+
return {
|
| 685 |
+
"available": False,
|
| 686 |
+
"configured": False,
|
| 687 |
+
"mode": "local_only",
|
| 688 |
+
}
|
| 689 |
+
|
| 690 |
+
try:
|
| 691 |
+
stats = hub.get_storage_stats()
|
| 692 |
+
return {
|
| 693 |
+
"available": True,
|
| 694 |
+
"configured": True,
|
| 695 |
+
**stats,
|
| 696 |
+
}
|
| 697 |
+
except Exception as e:
|
| 698 |
+
return {
|
| 699 |
+
"available": False,
|
| 700 |
+
"configured": True,
|
| 701 |
+
"error": str(e),
|
| 702 |
+
}
|
| 703 |
+
|
| 704 |
+
|
| 705 |
+
def delete_from_hub(path: str) -> dict[str, Any]:
|
| 706 |
+
"""Delete a file from Hub storage only (keeps local).
|
| 707 |
+
|
| 708 |
+
Args:
|
| 709 |
+
path: File path to delete from Hub
|
| 710 |
+
|
| 711 |
+
Returns:
|
| 712 |
+
Operation result
|
| 713 |
+
"""
|
| 714 |
+
hub = _get_hub_storage()
|
| 715 |
+
if not hub:
|
| 716 |
+
return {"success": False, "error": "Hub storage not configured"}
|
| 717 |
+
|
| 718 |
+
try:
|
| 719 |
+
success = hub.delete_file(path, commit_message=f"Delete: {path}")
|
| 720 |
+
return {"success": success, "path": path}
|
| 721 |
+
except Exception as e:
|
| 722 |
+
return {"success": False, "error": str(e)}
|