Z User commited on
Commit
817e3da
·
1 Parent(s): d234c81

☁️ Integrate Hugging Face Hub Cloud Storage

Browse files

NEW FEATURE: AI can now write code directly to HF Hub!

MAJOR ADDITIONS:
- code/hub_storage.py: Complete cloud storage module
- HubStorage class with full CRUD operations
- Support for local, hub, and hybrid modes
- Automatic sync between local and cloud
- Dataset repo support for persistent Spaces storage
- Batch operations for efficiency
- File caching and metadata tracking

- Updated code/tools/fs.py: Transparent Hub integration
- read_file: Falls back to Hub if not local
- write_file: Auto-syncs to Hub after local write
- edit_file/multi_edit: Syncs changes back
- list_dir/glob/grep: Merges local + Hub files
- New functions: sync_to_hub, sync_from_hub, get_hub_status

- Updated code/server/routes.py: 11 new API endpoints
- /api/hub_status: Check Hub configuration status
- /api/hub_init: Initialize/configure Hub storage
- /api/hub_sync_push: Push files to Hub
- /api/hub_sync_pull: Pull files from Hub
- /api/hub_full_sync: Bidirectional sync
- /api/hub_list_files: List Hub files
- /api/hub_read_file: Read from Hub only
- /api/hub_write_file: Write to Hub directly (AI uses this!)
- /api/hub_delete_file: Delete from Hub
- /api/hub_create_repo: Create new repo on Hub
- /api/hub_stats: Storage statistics

- Updated app.py: Auto-initialization
- Detects HF Spaces environment automatically
- Uses SPACE_ID and HF_TOKEN env vars
- Creates dataset repo for workspace persistence
- Graceful shutdown syncs to Hub
- Logs storage mode on startup

HOW IT WORKS:
1. On HF Spaces: Auto-detects environment
2. Creates/uses dataset repo for persistence
3. All file ops transparently sync to Hub
4. AI can call hub_write_file() to persist code
5. Workspace survives Space restarts!

CONFIGURATION:
- Set HF_TOKEN env variable (auto-detected)
- Optional: HUB_DATASET_REPO for custom storage
- Optional: HUB_STORAGE_MODE (local/hub/hybrid/auto)

Files changed (4) hide show
  1. app.py +114 -1
  2. code/hub_storage.py +933 -0
  3. code/server/routes.py +357 -0
  4. code/tools/fs.py +425 -81
app.py CHANGED
@@ -9,6 +9,8 @@ Enhanced version with:
9
  - Health check endpoint
10
  - Session management initialization
11
  - Model optimization on startup
 
 
12
 
13
  Blog: https://huggingface.co/blog/introducing-gradio-server
14
  """
@@ -16,6 +18,7 @@ Blog: https://huggingface.co/blog/introducing-gradio-server
16
  from __future__ import annotations
17
 
18
  import logging
 
19
  import signal
20
  import sys
21
  import time
@@ -38,6 +41,17 @@ def setup_signal_handlers(app):
38
  """Setup graceful shutdown handlers."""
39
  def shutdown_handler(signum, frame):
40
  logger.info("Received shutdown signal, cleaning up...")
 
 
 
 
 
 
 
 
 
 
 
41
  try:
42
  # Save sessions
43
  from code.server.session_manager import get_session_manager
@@ -54,7 +68,7 @@ def setup_signal_handlers(app):
54
  _unload_model()
55
  logger.info("Model unloaded")
56
  except Exception as e:
57
- logger.warning("Failed to unload model: %s", e)
58
 
59
  logger.info("Shutdown complete")
60
  sys.exit(0)
@@ -83,15 +97,107 @@ def initialize_session_manager():
83
  return None
84
 
85
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
86
  def main():
87
  start_time = time.time()
88
  logger.info("=" * 50)
89
  logger.info("SoniCoder v%s starting...", APP_VERSION)
90
  logger.info("=" * 50)
91
 
 
 
 
 
 
 
 
92
  # Initialize session manager
93
  session_mgr = initialize_session_manager()
94
 
 
 
 
95
  # Start loading default model in background
96
  from code.model.loader import start_background_load, add_progress_callback
97
 
@@ -137,6 +243,13 @@ def main():
137
  # Log startup time
138
  startup_time = time.time() - start_time
139
  logger.info("Startup completed in %.2f seconds", startup_time)
 
 
 
 
 
 
 
140
  logger.info("Launching...")
141
 
142
  # Launch the server
 
9
  - Health check endpoint
10
  - Session management initialization
11
  - Model optimization on startup
12
+ - **Hugging Face Hub cloud storage integration**
13
+ - Auto-detection for HF Spaces environment
14
 
15
  Blog: https://huggingface.co/blog/introducing-gradio-server
16
  """
 
18
  from __future__ import annotations
19
 
20
  import logging
21
+ import os
22
  import signal
23
  import sys
24
  import time
 
41
  """Setup graceful shutdown handlers."""
42
  def shutdown_handler(signum, frame):
43
  logger.info("Received shutdown signal, cleaning up...")
44
+
45
+ # Sync to Hub before shutdown if enabled
46
+ try:
47
+ from code.hub_storage import is_hub_available, get_hub_storage
48
+ if is_hub_available():
49
+ hub = get_hub_storage()
50
+ hub.sync_to_hub(commit_message="Auto-sync before shutdown")
51
+ logger.info("Workspace synced to Hub before shutdown")
52
+ except Exception as e:
53
+ logger.warning("Failed to sync to Hub: %s", e)
54
+
55
  try:
56
  # Save sessions
57
  from code.server.session_manager import get_session_manager
 
68
  _unload_model()
69
  logger.info("Model unloaded")
70
  except Exception as e:
71
+ logger.warning("Failed to unload model: %e", e)
72
 
73
  logger.info("Shutdown complete")
74
  sys.exit(0)
 
97
  return None
98
 
99
 
100
+ def initialize_hub_storage():
101
+ """
102
+ Initialize Hugging Face Hub cloud storage.
103
+
104
+ Auto-detects environment:
105
+ - On HF Spaces: Uses SPACE_ID, HF_TOKEN env vars
106
+ - Local: Requires explicit configuration or HF_TOKEN
107
+
108
+ Returns:
109
+ Initialized HubStorage instance or None if not configured.
110
+ """
111
+ try:
112
+ from code.hub_storage import init_hub_storage, is_hub_available
113
+
114
+ # Get configuration from environment
115
+ token = os.environ.get("HF_TOKEN", "") or os.environ.get("HF_AUTH_TOKEN", "")
116
+ space_id = os.environ.get("SPACE_ID", "")
117
+
118
+ # Determine mode based on environment
119
+ if space_id:
120
+ # Running on HF Spaces
121
+ logger.info("Detected HF Spaces environment: %s", space_id)
122
+
123
+ # Extract repo_id from space_id (format: "user/space-name")
124
+ if space_id and token:
125
+ # Use a dataset repo for workspace persistence
126
+ # This allows the AI's code to persist across Space restarts
127
+ user_name = space_id.split("/")[0] if "/" in space_id else "unknown"
128
+ dataset_repo_id = os.environ.get(
129
+ "HUB_DATASET_REPO",
130
+ f"{user_name}/sonicoder-workspace"
131
+ )
132
+
133
+ mode = os.environ.get("HUB_STORAGE_MODE", "hybrid")
134
+
135
+ storage = init_hub_storage(
136
+ token=token,
137
+ repo_id=space_id,
138
+ dataset_repo_id=dataset_repo_id,
139
+ mode=mode,
140
+ )
141
+
142
+ stats = storage.get_storage_stats()
143
+ logger.info(
144
+ "Hub storage initialized: mode=%s, hub=%s, dataset=%s",
145
+ stats["mode"],
146
+ stats["hub_enabled"],
147
+ stats["dataset_enabled"]
148
+ )
149
+
150
+ return storage
151
+ else:
152
+ logger.warning("HF Spaces detected but no token provided, using local storage only")
153
+ return None
154
+ elif token:
155
+ # Local mode with token
156
+ repo_id = os.environ.get("HUB_REPO_ID", "")
157
+ dataset_repo_id = os.environ.get("HUB_DATASET_REPO", "")
158
+
159
+ if repo_id or dataset_repo_id:
160
+ storage = init_hub_storage(
161
+ token=token,
162
+ repo_id=repo_id,
163
+ dataset_repo_id=dataset_repo_id,
164
+ mode="hybrid",
165
+ )
166
+
167
+ logger.info("Hub storage initialized in local mode")
168
+ return storage
169
+
170
+ # No configuration available
171
+ logger.info("Hub storage not configured (set HF_TOKEN to enable)")
172
+ return None
173
+
174
+ except ImportError:
175
+ logger.debug("huggingface_hub not installed, skipping Hub storage")
176
+ return None
177
+ except Exception as exc:
178
+ logger.warning("Hub storage initialization failed: %s", exc)
179
+ return None
180
+
181
+
182
  def main():
183
  start_time = time.time()
184
  logger.info("=" * 50)
185
  logger.info("SoniCoder v%s starting...", APP_VERSION)
186
  logger.info("=" * 50)
187
 
188
+ # Detect and log environment
189
+ space_id = os.environ.get("SPACE_ID", None)
190
+ if space_id:
191
+ logger.info("Running on Hugging Face Spaces: %s", space_id)
192
+ else:
193
+ logger.info("Running in local mode")
194
+
195
  # Initialize session manager
196
  session_mgr = initialize_session_manager()
197
 
198
+ # Initialize Hub storage (cloud persistence)
199
+ hub_storage = initialize_hub_storage()
200
+
201
  # Start loading default model in background
202
  from code.model.loader import start_background_load, add_progress_callback
203
 
 
243
  # Log startup time
244
  startup_time = time.time() - start_time
245
  logger.info("Startup completed in %.2f seconds", startup_time)
246
+
247
+ # Log storage info
248
+ if hub_storage:
249
+ logger.info("Cloud storage: ENABLED (files will persist on HF Hub)")
250
+ else:
251
+ logger.info("Cloud storage: DISABLED (local files only)")
252
+
253
  logger.info("Launching...")
254
 
255
  # Launch the server
code/hub_storage.py ADDED
@@ -0,0 +1,933 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Hugging Face Hub Cloud Storage Integration.
2
+
3
+ This module provides seamless integration with Hugging Face Hub for:
4
+ - Cloud-based file storage and retrieval
5
+ - Persistent workspace on HF Spaces
6
+ - Direct code writing to repositories
7
+ - Dataset-backed persistence
8
+
9
+ Architecture:
10
+ ┌─────────────┐ ┌──────────────────┐ ┌─────────────────┐
11
+ │ AI Agent │────▶│ hub_storage.py │────▶│ HuggingFace Hub │
12
+ │ │ │ (this module) │ │ (Cloud Storage) │
13
+ └─────────────┘ └──────────────────┘ └─────────────────┘
14
+
15
+
16
+ ┌──────────────────┐
17
+ │ Local Fallback │
18
+ │ (when offline) │
19
+ └──────────────────┘
20
+
21
+ Usage Modes:
22
+ 1. LOCAL: Files stored in ./workspace/ (default)
23
+ 2. HUB: Files synced to HF Hub repository
24
+ 3. HYBRID: Local + Hub sync (recommended for Spaces)
25
+ """
26
+
27
+ from __future__ import annotations
28
+
29
+ import json
30
+ import logging
31
+ import os
32
+ import tempfile
33
+ import time
34
+ from dataclasses import dataclass, field
35
+ from pathlib import Path
36
+ from typing import Any, Callable, Optional, Iterator
37
+ from contextlib import contextmanager
38
+
39
+ logger = logging.getLogger(__name__)
40
+
41
+
42
+ @dataclass
43
+ class HubConfig:
44
+ """Configuration for HF Hub storage."""
45
+
46
+ # Repository settings
47
+ repo_id: str = "" # e.g., "username/repo-name" or "username/workspace-data"
48
+ repo_type: str = "space" # "space", "model", or "dataset"
49
+
50
+ # Authentication
51
+ token: str = ""
52
+
53
+ # Storage mode
54
+ mode: str = "local" # "local", "hub", or "hybrid"
55
+
56
+ # Workspace path (for local/hybrid)
57
+ local_workspace: str = "./workspace"
58
+
59
+ # Sync settings
60
+ auto_sync: bool = True
61
+ sync_interval: int = 60 # seconds
62
+
63
+ # Dataset repo for persistent storage (recommended for Spaces)
64
+ dataset_repo_id: str = "" # e.g., "username/sonicoder-workspace"
65
+
66
+ def is_hub_enabled(self) -> bool:
67
+ """Check if Hub storage is enabled."""
68
+ return self.mode in ("hub", "hybrid") and bool(self.repo_id) and bool(self.token)
69
+
70
+ def is_dataset_storage_enabled(self) -> bool:
71
+ """Check if dataset-based storage is enabled."""
72
+ return bool(self.dataset_repo_id) and bool(self.token)
73
+
74
+
75
+ @dataclass
76
+ class FileMetadata:
77
+ """Metadata for a file in storage."""
78
+ path: str
79
+ size: int = 0
80
+ modified_at: float = field(default_factory=time.time)
81
+ created_at: float = field(default_factory=time.time)
82
+ content_hash: str = ""
83
+ is_local: bool = True
84
+ is_remote: bool = False
85
+
86
+
87
+ class HubStorageError(Exception):
88
+ """Custom exception for Hub storage operations."""
89
+
90
+ def __init__(self, message: str, operation: str = "", recoverable: bool = True):
91
+ self.message = message
92
+ self.operation = operation
93
+ self.recoverable = recoverable
94
+ super().__init__(f"[{operation}] {message}")
95
+
96
+
97
+ class HubStorage:
98
+ """
99
+ Main class for Hugging Face Hub cloud storage integration.
100
+
101
+ Provides transparent file operations that work with both
102
+ local filesystem and HF Hub, with automatic sync.
103
+
104
+ Example:
105
+ >>> storage = HubStorage(
106
+ ... repo_id="sonic-coder/sonicoder",
107
+ ... token="hf_...",
108
+ ... mode="hybrid",
109
+ ... dataset_repo_id="sonic-coder/sonicoder-workspace"
110
+ ... )
111
+ >>> storage.initialize()
112
+ >>> storage.write_file("app.py", "print('Hello!')")
113
+ >>> content = storage.read_file("app.py")
114
+ >>> files = storage.list_files()
115
+ """
116
+
117
+ def __init__(
118
+ self,
119
+ config: Optional[HubConfig] = None,
120
+ **kwargs
121
+ ):
122
+ self.config = config or HubConfig(**kwargs)
123
+ self._api = None
124
+ self._initialized = False
125
+ self._local_workspace: Path = Path(self.config.local_workspace)
126
+ self._sync_lock_needed = False
127
+
128
+ # Cache for remote files
129
+ self._remote_cache: dict[str, FileMetadata] = {}
130
+ self._cache_timestamp: float = 0
131
+ self._cache_ttl: int = 30 # seconds
132
+
133
+ # Operation callbacks
134
+ self._on_sync: Optional[Callable] = None
135
+ self._on_error: Optional[Callable] = None
136
+
137
+ @property
138
+ def api(self):
139
+ """Lazy initialization of HfApi."""
140
+ if self._api is None:
141
+ try:
142
+ from huggingface_hub import HfApi
143
+ self._api = HfApi(token=self.config.token)
144
+ except ImportError:
145
+ raise HubStorageError(
146
+ "huggingface_hub package not installed",
147
+ operation="initialize",
148
+ recoverable=False
149
+ )
150
+ return self._api
151
+
152
+ def initialize(self) -> dict[str, Any]:
153
+ """
154
+ Initialize the Hub storage system.
155
+
156
+ Creates necessary directories, validates connection,
157
+ and sets up the storage backend.
158
+
159
+ Returns:
160
+ Dict with initialization status and info.
161
+ """
162
+ result = {
163
+ "success": False,
164
+ "mode": self.config.mode,
165
+ "hub_enabled": False,
166
+ "dataset_enabled": False,
167
+ "workspace_path": str(self._local_workspace),
168
+ }
169
+
170
+ try:
171
+ # Create local workspace directory
172
+ self._local_workspace.mkdir(parents=True, exist_ok=True)
173
+
174
+ # Validate Hub connection if enabled
175
+ if self.config.is_hub_enabled():
176
+ try:
177
+ # Test API connection by getting user info
178
+ user_info = self.api.whoami()
179
+ result["hub_enabled"] = True
180
+ result["user"] = user_info.get("name", "unknown")
181
+ logger.info("Connected to HF Hub as: %s", result["user"])
182
+
183
+ # Ensure repo exists
184
+ self._ensure_repo_exists()
185
+
186
+ except Exception as e:
187
+ logger.warning("Hub connection failed: %s", e)
188
+ result["hub_error"] = str(e)
189
+ # Fall back to local mode
190
+ self.config.mode = "local"
191
+
192
+ # Check dataset storage
193
+ if self.config.is_dataset_storage_enabled():
194
+ try:
195
+ self._ensure_dataset_repo_exists()
196
+ result["dataset_enabled"] = True
197
+ logger.info("Dataset storage ready: %s", self.config.dataset_repo_id)
198
+ except Exception as e:
199
+ logger.warning("Dataset setup failed: %s", e)
200
+ result["dataset_error"] = str(e)
201
+
202
+ self._initialized = True
203
+ result["success"] = True
204
+
205
+ logger.info(
206
+ "HubStorage initialized: mode=%s, hub=%s, dataset=%s",
207
+ self.config.mode,
208
+ result["hub_enabled"],
209
+ result["dataset_enabled"]
210
+ )
211
+
212
+ except Exception as e:
213
+ logger.exception("Failed to initialize HubStorage")
214
+ result["error"] = str(e)
215
+
216
+ return result
217
+
218
+ def _ensure_repo_exists(self) -> None:
219
+ """Ensure the target repository exists."""
220
+ try:
221
+ self.api.repo_info(
222
+ repo_id=self.config.repo_id,
223
+ repo_type=self.config.repo_type
224
+ )
225
+ except Exception:
226
+ # Repo doesn't exist, create it
227
+ logger.info("Creating repo: %s", self.config.repo_id)
228
+ self.api.create_repo(
229
+ repo_id=self.config.repo_id,
230
+ repo_type=self.config.repo_type,
231
+ exist_ok=True,
232
+ private=False
233
+ )
234
+
235
+ def _ensure_dataset_repo_exists(self) -> None:
236
+ """Ensure the dataset repository exists for persistent storage."""
237
+ try:
238
+ self.api.repo_info(
239
+ repo_id=self.config.dataset_repo_id,
240
+ repo_type="dataset"
241
+ )
242
+ except Exception:
243
+ logger.info("Creating dataset repo: %s", self.config.dataset_repo_id)
244
+ self.api.create_repo(
245
+ repo_id=self.config.dataset_repo_id,
246
+ repo_type="dataset",
247
+ exist_ok=True,
248
+ private=True # Workspaces are usually private
249
+ )
250
+
251
+ # ─── File Operations ──────────────────────────────────────────────
252
+
253
+ def write_file(
254
+ self,
255
+ path: str,
256
+ content: str | bytes,
257
+ commit_message: str | None = None,
258
+ sync_to_hub: bool = True
259
+ ) -> FileMetadata:
260
+ """
261
+ Write a file to storage.
262
+
263
+ In hybrid/local mode, writes to local filesystem first.
264
+ If sync_to_hub and hub is enabled, also uploads to HF Hub.
265
+
266
+ Args:
267
+ path: Relative file path (e.g., "src/app.py")
268
+ content: File content (string or bytes)
269
+ commit_message: Optional git commit message
270
+ sync_to_hub: Whether to sync to Hub immediately
271
+
272
+ Returns:
273
+ FileMetadata with file information.
274
+ """
275
+ # Normalize path
276
+ path = self._normalize_path(path)
277
+
278
+ # Generate commit message if not provided
279
+ if not commit_message:
280
+ commit_message = f"Update {path}"
281
+
282
+ # Write to local filesystem first
283
+ local_path = self._local_workspace / path
284
+ local_path.parent.mkdir(parents=True, exist_ok=True)
285
+
286
+ if isinstance(content, bytes):
287
+ local_path.write_bytes(content)
288
+ else:
289
+ local_path.write_text(content, encoding='utf-8')
290
+
291
+ # Create metadata
292
+ metadata = FileMetadata(
293
+ path=path,
294
+ size=local_path.stat().st_size,
295
+ is_local=True
296
+ )
297
+
298
+ # Sync to Hub if enabled
299
+ if sync_to_hub and self._should_sync_to_hub():
300
+ try:
301
+ metadata = self._upload_to_hub(path, content, commit_message)
302
+ metadata.is_remote = True
303
+ except Exception as e:
304
+ logger.warning("Failed to sync %s to Hub: %s", path, e)
305
+ if self._on_error:
306
+ self._on_error("write", path, e)
307
+
308
+ return metadata
309
+
310
+ def read_file(self, path: str, use_cache: bool = True) -> str:
311
+ """
312
+ Read a file from storage.
313
+
314
+ Tries local first, then falls back to Hub if available.
315
+
316
+ Args:
317
+ path: Relative file path
318
+ use_cache: Whether to use cached remote file list
319
+
320
+ Returns:
321
+ File content as string.
322
+
323
+ Raises:
324
+ HubStorageError if file not found anywhere.
325
+ """
326
+ path = self._normalize_path(path)
327
+
328
+ # Try local first
329
+ local_path = self._local_workspace / path
330
+ if local_path.exists():
331
+ return local_path.read_text(encoding='utf-8')
332
+
333
+ # Try Hub if enabled
334
+ if self.config.is_hub_enabled() or self.config.is_dataset_storage_enabled():
335
+ try:
336
+ return self._download_from_hub(path)
337
+ except Exception as e:
338
+ logger.debug("Could not download %s from Hub: %s", path, e)
339
+
340
+ raise HubStorageError(
341
+ f"File not found: {path}",
342
+ operation="read_file",
343
+ recoverable=False
344
+ )
345
+
346
+ def read_binary(self, path: str) -> bytes:
347
+ """Read a file as binary."""
348
+ path = self._normalize_path(path)
349
+
350
+ local_path = self._local_workspace / path
351
+ if local_path.exists():
352
+ return local_path.read_bytes()
353
+
354
+ if self.config.is_hub_enabled():
355
+ # Download to temp and read
356
+ with tempfile.NamedTemporaryFile(delete=False) as tmp:
357
+ tmp_path = tmp.name
358
+
359
+ try:
360
+ self.api.hub_download(
361
+ repo_id=self._get_target_repo(),
362
+ filename=path,
363
+ repo_type=self._get_target_repo_type(),
364
+ local_dir=os.path.dirname(tmp_path),
365
+ )
366
+
367
+ # Find downloaded file
368
+ downloaded = Path(tmp_path).parent / Path(path).name
369
+ if downloaded.exists():
370
+ return downloaded.read_bytes()
371
+ finally:
372
+ # Cleanup temp
373
+ if os.path.exists(tmp_path):
374
+ os.unlink(tmp_path)
375
+
376
+ raise HubStorageError(f"Binary file not found: {path}", operation="read_binary")
377
+
378
+ def delete_file(self, path: str, commit_message: str | None = None) -> bool:
379
+ """
380
+ Delete a file from storage.
381
+
382
+ Args:
383
+ path: Relative file path
384
+ commit_message: Optional commit message
385
+
386
+ Returns:
387
+ True if deleted successfully.
388
+ """
389
+ path = self._normalize_path(path)
390
+
391
+ if not commit_message:
392
+ commit_message = f"Delete {path}"
393
+
394
+ success = True
395
+
396
+ # Delete from local
397
+ local_path = self._local_workspace / path
398
+ if local_path.exists():
399
+ local_path.unlink()
400
+ # Remove empty parent directories
401
+ self._cleanup_empty_dirs(local_path.parent)
402
+
403
+ # Delete from Hub
404
+ if self._should_sync_to_hub():
405
+ try:
406
+ self.api.delete_file(
407
+ path_in_repo=path,
408
+ repo_id=self._get_target_repo(),
409
+ repo_type=self._get_target_repo_type(),
410
+ commit_message=commit_message
411
+ )
412
+ except Exception as e:
413
+ logger.warning("Failed to delete %s from Hub: %s", path, e)
414
+ success = False
415
+
416
+ return success
417
+
418
+ def file_exists(self, path: str) -> bool:
419
+ """Check if a file exists in storage."""
420
+ path = self._normalize_path(path)
421
+
422
+ # Check local
423
+ if (self._local_workspace / path).exists():
424
+ return True
425
+
426
+ # Check Hub cache or remote
427
+ if self.config.is_hub_enabled():
428
+ try:
429
+ files = self.list_files(use_cache=True)
430
+ return any(f["path"] == path for f in files)
431
+ except Exception:
432
+ pass
433
+
434
+ return False
435
+
436
+ def list_files(
437
+ self,
438
+ path: str = ".",
439
+ recursive: bool = True,
440
+ use_cache: bool = True
441
+ ) -> list[dict[str, Any]]:
442
+ """
443
+ List files in storage.
444
+
445
+ Combines local and remote file listings.
446
+
447
+ Args:
448
+ path: Directory path to list (default: root)
449
+ recursive: Whether to list recursively
450
+ use_cache: Whether to use cached file listing
451
+
452
+ Returns:
453
+ List of file info dicts with 'path', 'size', 'modified' keys.
454
+ """
455
+ files = []
456
+ seen_paths = set()
457
+
458
+ # List local files
459
+ base_path = self._local_workspace / path
460
+ if base_path.exists():
461
+ for file_path in base_path.rglob("*") if recursive else base_path.iterdir():
462
+ if file_path.is_file():
463
+ rel_path = str(file_path.relative_to(self._local_workspace))
464
+ stat = file_path.stat()
465
+ files.append({
466
+ "path": rel_path.replace(os.sep, "/"),
467
+ "size": stat.st_size,
468
+ "modified": stat.st_mtime,
469
+ "is_local": True,
470
+ "is_remote": False,
471
+ })
472
+ seen_paths.add(rel_path)
473
+
474
+ # List remote files if hub enabled
475
+ if self.config.is_hub_enabled() or self.config.is_dataset_storage_enabled():
476
+ try:
477
+ remote_files = self._list_hub_files(use_cache=use_cache)
478
+ for rf in remote_files:
479
+ if rf["path"] not in seen_paths:
480
+ rf["is_local"] = False
481
+ rf["is_remote"] = True
482
+ files.append(rf)
483
+ except Exception as e:
484
+ logger.warning("Failed to list remote files: %e", e)
485
+
486
+ # Sort by path
487
+ files.sort(key=lambda x: x["path"])
488
+
489
+ return files
490
+
491
+ # ─── Hub Operations ────────────────────────────────────────────────
492
+
493
+ def _should_sync_to_hub(self) -> bool:
494
+ """Check if we should sync to Hub based on config."""
495
+ return (
496
+ self.config.is_hub_enabled() or
497
+ self.config.is_dataset_storage_enabled()
498
+ ) and self.config.auto_sync
499
+
500
+ def _get_target_repo(self) -> str:
501
+ """Get the target repository ID for operations."""
502
+ # Prefer dataset repo for workspace files
503
+ if self.config.is_dataset_storage_enabled():
504
+ return self.config.dataset_repo_id
505
+ return self.config.repo_id
506
+
507
+ def _get_target_repo_type(self) -> str:
508
+ """Get the target repository type."""
509
+ if self.config.is_dataset_storage_enabled():
510
+ return "dataset"
511
+ return self.config.repo_type
512
+
513
+ def _upload_to_hub(
514
+ self,
515
+ path: str,
516
+ content: str | bytes,
517
+ commit_message: str
518
+ ) -> FileMetadata:
519
+ """Upload a file to HF Hub."""
520
+
521
+ # For large content, use temp file
522
+ if isinstance(content, str) and len(content) > 100000: # > 100KB
523
+ with tempfile.NamedTemporaryFile(
524
+ mode='w',
525
+ suffix=Path(path).suffix,
526
+ delete=False,
527
+ encoding='utf-8'
528
+ ) as tmp:
529
+ tmp.write(content)
530
+ tmp_path = tmp.name
531
+
532
+ try:
533
+ self.api.upload_file(
534
+ path_or_fileobj=tmp_path,
535
+ path_in_repo=path,
536
+ repo_id=self._get_target_repo(),
537
+ repo_type=self._get_target_repo_type(),
538
+ commit_message=commit_message
539
+ )
540
+ finally:
541
+ os.unlink(tmp_path)
542
+ else:
543
+ # Upload directly from content
544
+ content_bytes = content.encode('utf-8') if isinstance(content, str) else content
545
+
546
+ from io import BytesIO
547
+ self.api.upload_file(
548
+ path_or_fileobj=BytesIO(content_bytes),
549
+ path_in_repo=path,
550
+ repo_id=self._get_target_repo(),
551
+ repo_type=self._get_target_repo_type(),
552
+ commit_message=commit_message
553
+ )
554
+
555
+ logger.info("Uploaded %s to Hub (%s)", path, self._get_target_repo())
556
+
557
+ return FileMetadata(
558
+ path=path,
559
+ size=len(content) if isinstance(content, str) else len(content_bytes),
560
+ is_remote=True,
561
+ is_local=(self._local_workspace / path).exists()
562
+ )
563
+
564
+ def _download_from_hub(self, path: str) -> str:
565
+ """Download a file from HF Hub."""
566
+
567
+ # Use hf_hub_download for caching
568
+ from huggingface_hub import hf_hub_download
569
+
570
+ local_path = hf_hub_download(
571
+ repo_id=self._get_target_repo(),
572
+ filename=path,
573
+ repo_type=self._get_target_repo_type(),
574
+ token=self.config.token
575
+ )
576
+
577
+ # Read and return content
578
+ with open(local_path, 'r', encoding='utf-8') as f:
579
+ return f.read()
580
+
581
+ def _list_hub_files(
582
+ self,
583
+ use_cache: bool = True
584
+ ) -> list[dict[str, Any]]:
585
+ """List files in the Hub repository."""
586
+
587
+ # Check cache
588
+ if use_cache and self._remote_cache:
589
+ if time.time() - self._cache_timestamp < self._cache_ttl:
590
+ return list(self._remote_cache.values())
591
+
592
+ # Fetch from API
593
+ repo_info = self.api.repo_info(
594
+ repo_id=self._get_target_repo(),
595
+ repo_type=self._get_target_repo_type()
596
+ )
597
+
598
+ files = []
599
+ siblings = getattr(repo_info, 'siblings', [])
600
+
601
+ for sibling in siblings:
602
+ if hasattr(sibling, 'rfilename'):
603
+ files.append({
604
+ "path": sibling.rfilename,
605
+ "size": getattr(sibling, 'size', 0),
606
+ "blob_id": getattr(sibling, 'blob_id', ''),
607
+ "is_local": False,
608
+ "is_remote": True,
609
+ })
610
+
611
+ # Update cache
612
+ self._remote_cache = {f["path"]: f for f in files}
613
+ self._cache_timestamp = time.time()
614
+
615
+ return files
616
+
617
+ # ─── Sync Operations ──────────────────────────────────────────────
618
+
619
+ def sync_to_hub(
620
+ self,
621
+ path: str | None = None,
622
+ commit_message: str = "Sync workspace"
623
+ ) -> dict[str, Any]:
624
+ """
625
+ Sync local files to Hub.
626
+
627
+ Args:
628
+ path: Specific file or directory to sync (None = all)
629
+ commit_message: Commit message for changes
630
+
631
+ Returns:
632
+ Dict with sync results.
633
+ """
634
+ result = {
635
+ "success": False,
636
+ "uploaded": [],
637
+ "skipped": [],
638
+ "errors": []
639
+ }
640
+
641
+ if not self.config.is_hub_enabled() and not self.config.is_dataset_storage_enabled():
642
+ result["errors"].append("Hub storage not configured")
643
+ return result
644
+
645
+ try:
646
+ files_to_sync = []
647
+
648
+ if path:
649
+ # Sync specific file/directory
650
+ full_path = self._local_workspace / path
651
+ if full_path.is_file():
652
+ files_to_sync.append(path)
653
+ elif full_path.is_dir():
654
+ for fp in full_path.rglob("*"):
655
+ if fp.is_file():
656
+ files_to_sync.append(str(fp.relative_to(self._local_workspace)))
657
+ else:
658
+ # Sync all local files
659
+ for fp in self._local_workspace.rglob("*"):
660
+ if fp.is_file():
661
+ rel = str(fp.relative_to(self._local_workspace))
662
+ files_to_sync.append(rel)
663
+
664
+ # Upload each file
665
+ for file_path in files_to_sync:
666
+ try:
667
+ local_path = self._local_workspace / file_path
668
+ content = local_path.read_text(encoding='utf-8')
669
+
670
+ self._upload_to_hub(
671
+ file_path,
672
+ content,
673
+ f"{commit_message}: {file_path}"
674
+ )
675
+
676
+ result["uploaded"].append(file_path)
677
+
678
+ except Exception as e:
679
+ logger.error("Failed to sync %s: %s", file_path, e)
680
+ result["errors"].append({"file": file_path, "error": str(e)})
681
+
682
+ result["success"] = len(result["errors"]) == 0
683
+
684
+ if self._on_sync:
685
+ self._on_sync(result)
686
+
687
+ except Exception as e:
688
+ result["errors"].append({"file": "*", "error": str(e)})
689
+ logger.exception("Sync failed")
690
+
691
+ return result
692
+
693
+ def sync_from_hub(
694
+ self,
695
+ path: str | None = None
696
+ ) -> dict[str, Any]:
697
+ """
698
+ Sync files from Hub to local.
699
+
700
+ Args:
701
+ path: Specific file to download (None = all)
702
+
703
+ Returns:
704
+ Dict with sync results.
705
+ """
706
+ result = {
707
+ "success": False,
708
+ "downloaded": [],
709
+ "errors": []
710
+ }
711
+
712
+ try:
713
+ if path:
714
+ # Download specific file
715
+ content = self._download_from_hub(path)
716
+ local_path = self._local_workspace / path
717
+ local_path.parent.mkdir(parents=True, exist_ok=True)
718
+ local_path.write_text(content, encoding='utf-8')
719
+ result["downloaded"].append(path)
720
+ else:
721
+ # Download all files
722
+ remote_files = self._list_hub_files(use_cache=False)
723
+
724
+ for rf in remote_files:
725
+ try:
726
+ content = self._download_from_hub(rf["path"])
727
+ local_path = self._local_workspace / rf["path"]
728
+ local_path.parent.mkdir(parents=True, exist_ok=True)
729
+ local_path.write_text(content, encoding='utf-8')
730
+ result["downloaded"].append(rf["path"])
731
+ except Exception as e:
732
+ result["errors"].append({
733
+ "file": rf["path"],
734
+ "error": str(e)
735
+ })
736
+
737
+ result["success"] = len(result["errors"]) == len(remote_files) == 0 or len(remote_files) == 0
738
+
739
+ except Exception as e:
740
+ result["errors"].append({"file": "*", "error": str(e)})
741
+ logger.exception("Sync from hub failed")
742
+
743
+ return result
744
+
745
+ def full_sync(self) -> dict[str, Any]:
746
+ """
747
+ Perform bidirectional sync between local and Hub.
748
+
749
+ Merges changes from both sides.
750
+ """
751
+ result = {
752
+ "to_hub": {},
753
+ "from_hub": {}
754
+ }
755
+
756
+ # Push local to hub
757
+ result["to_hub"] = self.sync_to_hub(commit_message="Full sync push")
758
+
759
+ # Pull new/updated from hub
760
+ result["from_hub"] = self.sync_from_hub()
761
+
762
+ return result
763
+
764
+ # ─── Utility Methods ──────────────────────────────────────────────
765
+
766
+ def _normalize_path(self, path: str) -> str:
767
+ """Normalize a file path."""
768
+ # Remove leading ./
769
+ if path.startswith("./"):
770
+ path = path[2:]
771
+ # Convert backslashes
772
+ path = path.replace("\\", "/")
773
+ # Remove double slashes
774
+ while "//" in path:
775
+ path = path.replace("//", "/")
776
+ # Security: prevent path traversal
777
+ if ".." in path:
778
+ raise HubStorageError(
779
+ "Path traversal detected",
780
+ operation="_normalize_path",
781
+ recoverable=False
782
+ )
783
+ return path.lstrip("/")
784
+
785
+ def _cleanup_empty_dirs(self, dir_path: Path) -> None:
786
+ """Remove empty directories up to workspace root."""
787
+ try:
788
+ while dir_path != self._local_workspace:
789
+ if dir_path.is_dir() and not any(dir_path.iterdir()):
790
+ dir_path.rmdir()
791
+ dir_path = dir_path.parent
792
+ else:
793
+ break
794
+ except Exception:
795
+ pass
796
+
797
+ def get_storage_stats(self) -> dict[str, Any]:
798
+ """Get statistics about current storage."""
799
+ stats = {
800
+ "mode": self.config.mode,
801
+ "hub_enabled": self.config.is_hub_enabled(),
802
+ "dataset_enabled": self.config.is_dataset_storage_enabled(),
803
+ "local_files": 0,
804
+ "local_size_mb": 0,
805
+ "last_sync": self._cache_timestamp,
806
+ }
807
+
808
+ # Count local files
809
+ if self._local_workspace.exists():
810
+ for f in self._local_workspace.rglob("*"):
811
+ if f.is_file():
812
+ stats["local_files"] += 1
813
+ stats["local_size_mb"] += f.stat().st_size / (1024 * 1024)
814
+
815
+ stats["local_size_mb"] = round(stats["local_size_mb"], 2)
816
+
817
+ return stats
818
+
819
+ def clear_cache(self) -> None:
820
+ """Clear the remote file cache."""
821
+ self._remote_cache = {}
822
+ self._cache_timestamp = 0
823
+
824
+ def set_callbacks(
825
+ self,
826
+ on_sync: Callable | None = None,
827
+ on_error: Callable | None = None
828
+ ) -> None:
829
+ """Set callback functions for events."""
830
+ self._on_sync = on_sync
831
+ self._on_error = on_error
832
+
833
+ @contextmanager
834
+ def batch_operation(self, commit_message: str = "Batch update"):
835
+ """
836
+ Context manager for batch operations.
837
+
838
+ All writes within the context are grouped into one commit.
839
+
840
+ Usage:
841
+ with storage.batch_operation("Create API"):
842
+ storage.write_file("app.py", "...")
843
+ storage.write_file("config.py", "...")
844
+ """
845
+ original_auto_sync = self.config.auto_sync
846
+ self.config.auto_sync = False
847
+
848
+ files_written = []
849
+
850
+ try:
851
+ yield files_written
852
+
853
+ # Commit all at once
854
+ if files_written:
855
+ self.sync_to_hub(commit_message=commit_message)
856
+ finally:
857
+ self.config.auto_sync = original_auto_sync
858
+
859
+
860
+ # ─── Global Instance Management ────────────────────────────────────────
861
+
862
+ _global_storage: Optional[HubStorage] = None
863
+
864
+
865
+ def get_hub_storage(**kwargs) -> HubStorage:
866
+ """Get or create the global HubStorage instance."""
867
+ global _global_storage
868
+ if _global_storage is None:
869
+ _global_storage = HubStorage(**kwargs)
870
+ return _global_storage
871
+
872
+
873
+ def init_hub_storage(
874
+ token: str = "",
875
+ repo_id: str = "",
876
+ dataset_repo_id: str = "",
877
+ mode: str = "auto",
878
+ **kwargs
879
+ ) -> HubStorage:
880
+ """
881
+ Initialize the global HubStorage instance.
882
+
883
+ Auto-detects environment when running on HF Spaces.
884
+
885
+ Args:
886
+ token: HF API token
887
+ repo_id: Main repository ID
888
+ dataset_repo_id: Dataset repository for workspace storage
889
+ mode: Storage mode ("local", "hub", "hybrid", "auto")
890
+
891
+ Returns:
892
+ Initialized HubStorage instance.
893
+ """
894
+ global _global_storage
895
+
896
+ # Auto-detect mode
897
+ if mode == "auto":
898
+ # Check if running on HF Spaces
899
+ if os.environ.get("SPACE_ID"):
900
+ mode = "hybrid" if token else "local"
901
+ else:
902
+ mode = "local"
903
+
904
+ # Get token from env if not provided
905
+ if not token:
906
+ token = os.environ.get("HF_TOKEN", "") or os.environ.get("HF_AUTH_TOKEN", "")
907
+
908
+ config = HubConfig(
909
+ token=token,
910
+ repo_id=repo_id,
911
+ dataset_repo_id=dataset_repo_id,
912
+ mode=mode,
913
+ **kwargs
914
+ )
915
+
916
+ _global_storage = HubStorage(config=config)
917
+ _global_storage.initialize()
918
+
919
+ return _global_storage
920
+
921
+
922
+ def is_hub_available() -> bool:
923
+ """Check if Hub storage is available and configured."""
924
+ if _global_storage is None:
925
+ return False
926
+ return _global_storage.config.is_hub_enabled()
927
+
928
+
929
+ def get_storage_mode() -> str:
930
+ """Get current storage mode."""
931
+ if _global_storage is None:
932
+ return "not_initialized"
933
+ return _global_storage.config.mode
code/server/routes.py CHANGED
@@ -1662,3 +1662,360 @@ def handle_push_github(
1662
  timeout=timeout_int,
1663
  )
1664
  yield json.dumps(result, default=str)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1662
  timeout=timeout_int,
1663
  )
1664
  yield json.dumps(result, default=str)
1665
+
1666
+
1667
+ # ═══════════════════════════════════════════════════════════════════════
1668
+ # HUB STORAGE API ENDPOINTS
1669
+ # ═══════════════════════════════════════════════════════════════════════
1670
+
1671
+ @app.api(name="hub_status")
1672
+ def handle_hub_status() -> str:
1673
+ """Get current Hub storage status and configuration.
1674
+
1675
+ Returns:
1676
+ JSON dict with: available, configured, mode, hub_enabled,
1677
+ dataset_enabled, local_files, local_size_mb, etc.
1678
+ """
1679
+ from code.tools.fs import get_hub_status
1680
+
1681
+ result = get_hub_status()
1682
+ yield json.dumps(result, default=str)
1683
+
1684
+
1685
+ @app.api(name="hub_init")
1686
+ def handle_hub_init(
1687
+ token: str = "",
1688
+ repo_id: str = "",
1689
+ dataset_repo_id: str = "",
1690
+ mode: str = "auto",
1691
+ ) -> str:
1692
+ """Initialize/configure Hub storage.
1693
+
1694
+ This sets up the connection to Hugging Face Hub for cloud storage.
1695
+
1696
+ Args:
1697
+ token: HF API token (or use HF_TOKEN env var)
1698
+ repo_id: Main repository ID (e.g., "user/repo")
1699
+ dataset_repo_id: Dataset repo for workspace persistence
1700
+ mode: "local", "hub", "hybrid", or "auto"
1701
+
1702
+ Returns:
1703
+ JSON dict with initialization status and config.
1704
+ """
1705
+ from code.hub_storage import init_hub_storage
1706
+
1707
+ try:
1708
+ storage = init_hub_storage(
1709
+ token=token,
1710
+ repo_id=repo_id,
1711
+ dataset_repo_id=dataset_repo_id,
1712
+ mode=mode,
1713
+ )
1714
+
1715
+ result = {
1716
+ "success": True,
1717
+ "message": "Hub storage initialized",
1718
+ "config": {
1719
+ "mode": storage.config.mode,
1720
+ "repo_id": storage.config.repo_id,
1721
+ "dataset_repo_id": storage.config.dataset_repo_id,
1722
+ "hub_enabled": storage.config.is_hub_enabled(),
1723
+ "dataset_enabled": storage.config.is_dataset_storage_enabled(),
1724
+ },
1725
+ "stats": storage.get_storage_stats(),
1726
+ }
1727
+ except Exception as e:
1728
+ result = {
1729
+ "success": False,
1730
+ "error": str(e),
1731
+ }
1732
+
1733
+ yield json.dumps(result, default=str)
1734
+
1735
+
1736
+ @app.api(name="hub_sync_push", concurrency_limit=1)
1737
+ def handle_hub_sync_push(path: str = "") -> str:
1738
+ """Sync local workspace files to HF Hub.
1739
+
1740
+ Args:
1741
+ path: Optional specific file/directory to sync
1742
+
1743
+ Returns:
1744
+ JSON dict with sync results.
1745
+ """
1746
+ from code.tools.fs import sync_to_hub
1747
+
1748
+ try:
1749
+ result = sync_to_hub(path=path if path else None)
1750
+ yield json.dumps(result, default=str)
1751
+ except Exception as e:
1752
+ yield json.dumps({"success": False, "error": str(e)}, default=str)
1753
+
1754
+
1755
+ @app.api(name="hub_sync_pull", concurrency_limit=1)
1756
+ def handle_hub_sync_pull(path: str = "") -> str:
1757
+ """Sync files from HF Hub to local workspace.
1758
+
1759
+ Args:
1760
+ path: Optional specific file to download
1761
+
1762
+ Returns:
1763
+ JSON dict with sync results.
1764
+ """
1765
+ from code.tools.fs import sync_from_hub
1766
+
1767
+ try:
1768
+ result = sync_from_hub(path=path if path else None)
1769
+ yield json.dumps(result, default=str)
1770
+ except Exception as e:
1771
+ yield json.dumps({"success": False, "error": str(e)}, default=str)
1772
+
1773
+
1774
+ @app.api(name="hub_full_sync", concurrency_limit=1)
1775
+ def handle_hub_full_sync() -> str:
1776
+ """Perform bidirectional sync between local and Hub.
1777
+
1778
+ Merges changes from both sides.
1779
+
1780
+ Returns:
1781
+ JSON dict with both push and pull results.
1782
+ """
1783
+ from code.hub_storage import get_hub_storage, is_hub_available
1784
+
1785
+ if not is_hub_available():
1786
+ yield json.dumps({
1787
+ "success": False,
1788
+ "error": "Hub storage not configured"
1789
+ }, default=str)
1790
+ return
1791
+
1792
+ try:
1793
+ hub = get_hub_storage()
1794
+ result = hub.full_sync()
1795
+ yield json.dumps(result, default=str)
1796
+ except Exception as e:
1797
+ yield json.dumps({"success": False, "error": str(e)}, default=str)
1798
+
1799
+
1800
+ @app.api(name="hub_list_files")
1801
+ def handle_hub_list_files(path: str = ".", recursive: bool = True) -> str:
1802
+ """List files in Hub storage.
1803
+
1804
+ Args:
1805
+ path: Directory path to list
1806
+ recursive: Whether to list recursively
1807
+
1808
+ Returns:
1809
+ JSON dict with file listing.
1810
+ """
1811
+ from code.hub_storage import get_hub_storage, is_hub_available
1812
+
1813
+ if not is_hub_available():
1814
+ yield json.dumps({
1815
+ "success": False,
1816
+ "error": "Hub storage not configured",
1817
+ "files": []
1818
+ }, default_str=str)
1819
+ return
1820
+
1821
+ try:
1822
+ hub = get_hub_storage()
1823
+ files = hub.list_files(path=path, recursive=recursive)
1824
+ yield json.dumps({
1825
+ "success": True,
1826
+ "files": files,
1827
+ "count": len(files),
1828
+ }, default=str)
1829
+ except Exception as e:
1830
+ yield json.dumps({"success": False, "error": str(e), "files": []}, default=str)
1831
+
1832
+
1833
+ @app.api(name="hub_delete_file")
1834
+ def handle_hub_delete_file(path: str) -> str:
1835
+ """Delete a file from Hub storage.
1836
+
1837
+ Args:
1838
+ path: File path to delete
1839
+
1840
+ Returns:
1841
+ JSON dict with operation result.
1842
+ """
1843
+ from code.tools.fs import delete_from_hub
1844
+
1845
+ try:
1846
+ result = delete_from_hub(path)
1847
+ yield json.dumps(result, default=str)
1848
+ except Exception as e:
1849
+ yield json.dumps({"success": False, "error": str(e)}, default=str)
1850
+
1851
+
1852
+ @app.api(name="hub_read_file")
1853
+ def handle_hub_read_file(path: str) -> str:
1854
+ """Read a file specifically from Hub storage.
1855
+
1856
+ Args:
1857
+ path: File path to read
1858
+
1859
+ Returns:
1860
+ JSON dict with file content.
1861
+ """
1862
+ from code.hub_storage import get_hub_storage, is_hub_available
1863
+
1864
+ if not is_hub_available():
1865
+ yield json.dumps({
1866
+ "success": False,
1867
+ "error": "Hub storage not configured"
1868
+ }, default_str=str)
1869
+ return
1870
+
1871
+ try:
1872
+ hub = get_hub_storage()
1873
+ content = hub.read_file(path)
1874
+ yield json.dumps({
1875
+ "success": True,
1876
+ "path": path,
1877
+ "content": content,
1878
+ "size": len(content),
1879
+ }, default_str=str)
1880
+ except Exception as e:
1881
+ yield json.dumps({"success": False, "error": str(e)}, default_str=str)
1882
+
1883
+
1884
+ @app.api(name="hub_write_file")
1885
+ def handle_hub_write_file(path: str, content: str, commit_message: str = "") -> str:
1886
+ """Write a file directly to Hub storage.
1887
+
1888
+ This allows the AI to write code that persists on HF Hub.
1889
+
1890
+ Args:
1891
+ path: Destination file path (e.g., "src/app.py")
1892
+ content: File content
1893
+ commit_message: Optional commit message
1894
+
1895
+ Returns:
1896
+ JSON dict with operation result and metadata.
1897
+ """
1898
+ from code.hub_storage import get_hub_storage, is_hub_available
1899
+
1900
+ if not is_hub_available():
1901
+ yield json.dumps({
1902
+ "success": False,
1903
+ "error": "Hub storage not configured. Initialize with /hub_init first."
1904
+ }, default_str=str)
1905
+ return
1906
+
1907
+ try:
1908
+ hub = get_hub_storage()
1909
+
1910
+ # Use a descriptive commit message if not provided
1911
+ if not commit_message:
1912
+ commit_message = f"AI generated: {path}"
1913
+
1914
+ metadata = hub.write_file(
1915
+ path=path,
1916
+ content=content,
1917
+ commit_message=commit_message,
1918
+ sync_to_hub=True # Force immediate upload
1919
+ )
1920
+
1921
+ yield json.dumps({
1922
+ "success": True,
1923
+ "path": path,
1924
+ "size": metadata.size,
1925
+ "is_remote": metadata.is_remote,
1926
+ "is_local": metadata.is_local,
1927
+ "message": f"File written to {'Hub' if metadata.is_remote else 'local'} storage",
1928
+ }, default_str=str)
1929
+
1930
+ except Exception as e:
1931
+ yield json.dumps({"success": False, "error": str(e)}, default_str=str)
1932
+
1933
+
1934
+ @app.api(name="hub_create_repo")
1935
+ def handle_hub_create_repo(
1936
+ repo_id: str,
1937
+ repo_type: str = "dataset",
1938
+ private: bool = True,
1939
+ token: str = "",
1940
+ ) -> str:
1941
+ """Create a new repository on HF Hub for storage.
1942
+
1943
+ Args:
1944
+ repo_id: Repository ID (e.g., "username/my-workspace")
1945
+ repo_type: "space", "model", or "dataset"
1946
+ private: Whether to make it private
1947
+ token: HF API token
1948
+
1949
+ Returns:
1950
+ JSON dict with creation result.
1951
+ """
1952
+ from huggingface_hub import HfApi
1953
+
1954
+ try:
1955
+ api = HfApi(token=token) if token else HfApi()
1956
+
1957
+ url = api.create_repo(
1958
+ repo_id=repo_id,
1959
+ repo_type=repo_type,
1960
+ private=private,
1961
+ exist_ok=True,
1962
+ )
1963
+
1964
+ yield json.dumps({
1965
+ "success": True,
1966
+ "repo_id": repo_id,
1967
+ "repo_type": repo_type,
1968
+ "private": private,
1969
+ "url": url,
1970
+ "message": f"Repository {repo_id} created/verified",
1971
+ }, default_str=str)
1972
+
1973
+ except Exception as e:
1974
+ yield json.dumps({
1975
+ "success": False,
1976
+ "error": str(e),
1977
+ }, default_str=str)
1978
+
1979
+
1980
+ @app.api(name="hub_stats")
1981
+ def handle_hub_stats() -> str:
1982
+ """Get detailed statistics about current storage usage.
1983
+
1984
+ Returns:
1985
+ JSON dict with storage statistics for both local and Hub.
1986
+ """
1987
+ from code.hub_storage import get_hub_storage, is_hub_available
1988
+ from code.tools.fs import snapshot_workspace
1989
+
1990
+ stats = {
1991
+ "timestamp": __import__("time").time(),
1992
+ "local": {},
1993
+ "hub": None,
1994
+ }
1995
+
1996
+ # Local stats
1997
+ try:
1998
+ workspace_snapshot = snapshot_workspace()
1999
+ total_size = sum(len(c.encode('utf-8')) for c in workspace_snapshot.values())
2000
+ stats["local"] = {
2001
+ "file_count": len(workspace_snapshot),
2002
+ "total_size_bytes": total_size,
2003
+ "total_size_mb": round(total_size / (1024 * 1024), 2),
2004
+ }
2005
+ except Exception as e:
2006
+ stats["local"]["error"] = str(e)
2007
+
2008
+ # Hub stats
2009
+ if is_hub_available():
2010
+ try:
2011
+ hub = get_hub_storage()
2012
+ stats["hub"] = hub.get_storage_stats()
2013
+
2014
+ # Get remote file count
2015
+ hub_files = hub.list_files(use_cache=False)
2016
+ stats["hub"]["remote_file_count"] = len(hub_files)
2017
+
2018
+ except Exception as e:
2019
+ stats["hub"] = {"error": str(e)}
2020
+
2021
+ yield json.dumps(stats, default_str=str)
code/tools/fs.py CHANGED
@@ -1,5 +1,10 @@
1
  """File system tools: read, write, edit, list, glob, grep.
2
 
 
 
 
 
 
3
  All tools are sandboxed to a configurable workspace root (default: ./workspace).
4
  They return JSON-serializable dicts so they can be exposed via the API.
5
  """
@@ -7,11 +12,14 @@ They return JSON-serializable dicts so they can be exposed via the API.
7
  from __future__ import annotations
8
 
9
  import fnmatch
 
10
  import os
11
  import re
12
  import shutil
13
  from pathlib import Path
14
- from typing import Any
 
 
15
 
16
  # ─── Workspace sandbox ──────────────────────────────────────────────────
17
 
@@ -21,6 +29,9 @@ _DEFAULT_WORKSPACE = os.environ.get(
21
  os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "workspace")),
22
  )
23
 
 
 
 
24
 
25
  def get_workspace_root() -> str:
26
  """Return the absolute path of the agent's workspace root."""
@@ -49,11 +60,30 @@ def _resolve_safe(path: str) -> str:
49
  return full
50
 
51
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
52
  # ─── read_file ──────────────────────────────────────────────────────────
53
 
54
  def read_file(path: str, offset: int = 0, limit: int | None = None) -> dict[str, Any]:
55
  """Read a text file from the workspace.
56
 
 
 
57
  Args:
58
  path: Relative path inside the workspace, or absolute within it.
59
  offset: 1-indexed line to start reading from.
@@ -64,32 +94,62 @@ def read_file(path: str, offset: int = 0, limit: int | None = None) -> dict[str,
64
  """
65
  try:
66
  full = _resolve_safe(path)
67
- if not os.path.exists(full):
68
- return {"success": False, "error": f"File not found: {path}"}
69
- if os.path.isdir(full):
70
- return {"success": False, "error": f"Path is a directory: {path}"}
71
-
72
- with open(full, "r", encoding="utf-8", errors="replace") as f:
73
- lines = f.readlines()
74
-
75
- total = len(lines)
76
- start = max(0, (offset - 1) if offset > 0 else 0)
77
- end = (start + limit) if limit else total
78
- selected = lines[start:end]
79
-
80
- # Re-number for display
81
- numbered = "".join(
82
- f"{start + i + 1:6}\t{line}" for i, line in enumerate(selected)
83
- )
84
 
85
- return {
86
- "success": True,
87
- "path": path,
88
- "content": numbered,
89
- "line_count": total,
90
- "returned_lines": len(selected),
91
- "truncated": end < total,
92
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
93
  except Exception as exc:
94
  return {"success": False, "error": str(exc)}
95
 
@@ -97,17 +157,52 @@ def read_file(path: str, offset: int = 0, limit: int | None = None) -> dict[str,
97
  # ─── write_file ─────────────────────────────────────────────────────────
98
 
99
  def write_file(path: str, content: str) -> dict[str, Any]:
100
- """Write content to a file, creating parent directories as needed."""
 
 
 
 
 
 
 
 
 
 
101
  try:
102
  full = _resolve_safe(path)
103
  os.makedirs(os.path.dirname(full), exist_ok=True)
 
 
104
  with open(full, "w", encoding="utf-8") as f:
105
  f.write(content)
106
- return {
 
107
  "success": True,
108
  "path": path,
109
  "bytes_written": len(content.encode("utf-8")),
 
110
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
111
  except Exception as exc:
112
  return {"success": False, "error": str(exc)}
113
 
@@ -120,15 +215,23 @@ def edit_file(
120
  new_str: str,
121
  replace_all: bool = False,
122
  ) -> dict[str, Any]:
123
- """Replace occurrences of old_str with new_str in a file."""
 
 
 
124
  try:
125
  full = _resolve_safe(path)
126
- if not os.path.exists(full):
 
 
 
 
 
 
 
 
127
  return {"success": False, "error": f"File not found: {path}"}
128
 
129
- with open(full, "r", encoding="utf-8") as f:
130
- content = f.read()
131
-
132
  if old_str not in content:
133
  return {
134
  "success": False,
@@ -152,28 +255,49 @@ def edit_file(
152
  old_str, new_str, 1
153
  )
154
 
 
 
155
  with open(full, "w", encoding="utf-8") as f:
156
  f.write(new_content)
157
 
158
- return {
159
  "success": True,
160
  "path": path,
161
  "replacements": count,
162
  }
 
 
 
 
 
 
 
 
 
 
 
163
  except Exception as exc:
164
  return {"success": False, "error": str(exc)}
165
 
166
 
167
  def multi_edit(path: str, edits: list[dict[str, Any]]) -> dict[str, Any]:
168
- """Apply multiple edits to a file atomically (all-or-nothing)."""
 
 
 
169
  try:
170
  full = _resolve_safe(path)
171
- if not os.path.exists(full):
 
 
 
 
 
 
 
 
172
  return {"success": False, "error": f"File not found: {path}"}
173
 
174
- with open(full, "r", encoding="utf-8") as f:
175
- content = f.read()
176
-
177
  applied = 0
178
  for edit in edits:
179
  old_str = edit.get("old_str", "")
@@ -196,10 +320,23 @@ def multi_edit(path: str, edits: list[dict[str, Any]]) -> dict[str, Any]:
196
  )
197
  applied += 1
198
 
 
 
199
  with open(full, "w", encoding="utf-8") as f:
200
  f.write(content)
201
 
202
- return {"success": True, "path": path, "applied": applied}
 
 
 
 
 
 
 
 
 
 
 
203
  except Exception as exc:
204
  return {"success": False, "error": str(exc)}
205
 
@@ -207,30 +344,56 @@ def multi_edit(path: str, edits: list[dict[str, Any]]) -> dict[str, Any]:
207
  # ─── list_dir ───────────────────────────────────────────────────────────
208
 
209
  def list_dir(path: str = ".") -> dict[str, Any]:
210
- """List directory contents."""
 
 
 
211
  try:
212
  full = _resolve_safe(path)
213
- if not os.path.exists(full):
214
- return {"success": False, "error": f"Path not found: {path}"}
215
- if not os.path.isdir(full):
216
- return {"success": False, "error": f"Not a directory: {path}"}
217
-
218
  entries = []
219
- for name in sorted(os.listdir(full)):
220
- entry_path = os.path.join(full, name)
221
- stat = os.stat(entry_path)
222
- entries.append({
223
- "name": name,
224
- "type": "dir" if os.path.isdir(entry_path) else "file",
225
- "size": stat.st_size,
226
- "path": os.path.relpath(entry_path, get_workspace_root()),
227
- })
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
228
 
229
  return {
230
  "success": True,
231
  "path": path,
232
- "entries": entries,
233
  }
 
234
  except Exception as exc:
235
  return {"success": False, "error": str(exc)}
236
 
@@ -238,18 +401,43 @@ def list_dir(path: str = ".") -> dict[str, Any]:
238
  # ─── glob ───────────────────────────────────────────────────────────────
239
 
240
  def glob_paths(pattern: str, path: str = ".") -> dict[str, Any]:
241
- """Glob file paths matching a pattern, recursively."""
 
 
 
242
  try:
243
  full = _resolve_safe(path)
244
- matches: list[str] = []
245
- for root_dir, _dirs, files in os.walk(full):
246
- for fname in files:
247
- if fnmatch.fnmatch(fname, pattern) or fnmatch.fnmatch(
248
- os.path.relpath(os.path.join(root_dir, fname), full), pattern
249
- ):
250
- matches.append(os.path.relpath(os.path.join(root_dir, fname), get_workspace_root()))
251
- matches.sort()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
252
  return {"success": True, "pattern": pattern, "matches": matches}
 
253
  except Exception as exc:
254
  return {"success": False, "error": str(exc)}
255
 
@@ -263,26 +451,61 @@ def grep_search(
263
  ignore_case: bool = False,
264
  max_results: int = 100,
265
  ) -> dict[str, Any]:
266
- """Search file contents with a regex pattern."""
 
 
 
267
  try:
268
- full = _resolve_safe(path)
269
  flags = re.IGNORECASE if ignore_case else 0
270
  regex = re.compile(pattern, flags)
271
 
272
  matches: list[dict[str, Any]] = []
273
- for root_dir, _dirs, files in os.walk(full):
274
- for fname in files:
275
- if include and not fnmatch.fnmatch(fname, include):
276
- continue
277
- fpath = os.path.join(root_dir, fname)
278
- try:
279
- with open(fpath, "r", encoding="utf-8", errors="replace") as f:
280
- for lineno, line in enumerate(f, 1):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
281
  if regex.search(line):
282
  matches.append({
283
- "file": os.path.relpath(fpath, get_workspace_root()),
284
  "line": lineno,
285
  "text": line.rstrip()[:500],
 
286
  })
287
  if len(matches) >= max_results:
288
  return {
@@ -291,8 +514,10 @@ def grep_search(
291
  "matches": matches,
292
  "truncated": True,
293
  }
294
- except (UnicodeDecodeError, PermissionError):
295
- continue
 
 
296
 
297
  return {
298
  "success": True,
@@ -300,6 +525,7 @@ def grep_search(
300
  "matches": matches,
301
  "truncated": False,
302
  }
 
303
  except re.error as exc:
304
  return {"success": False, "error": f"Invalid regex: {exc}"}
305
  except Exception as exc:
@@ -309,7 +535,10 @@ def grep_search(
309
  # ─── Workspace management ───────────────────────────────────────────────
310
 
311
  def list_workspace_tree(max_depth: int = 3) -> dict[str, Any]:
312
- """Return a tree view of the workspace."""
 
 
 
313
  try:
314
  root = get_workspace_root()
315
 
@@ -333,13 +562,25 @@ def list_workspace_tree(max_depth: int = 3) -> dict[str, Any]:
333
  return {"name": os.path.basename(path), "type": "dir", "children": entries}
334
 
335
  tree = _walk(root, 0)
 
 
 
 
 
 
 
 
336
  return {"success": True, "tree": tree}
 
337
  except Exception as exc:
338
  return {"success": False, "error": str(exc)}
339
 
340
 
341
  def reset_workspace() -> dict[str, Any]:
342
- """Clear all files in the workspace (used by /new command)."""
 
 
 
343
  try:
344
  root = get_workspace_root()
345
  if os.path.exists(root):
@@ -357,12 +598,14 @@ def reset_workspace() -> dict[str, Any]:
357
  def snapshot_workspace() -> dict[str, str]:
358
  """Return a dict of {relative_path: content} for all text files in the workspace.
359
 
 
360
  Used to package workspace files for ZIP/HF deploy.
361
  """
362
  root = get_workspace_root()
363
  files: dict[str, str] = {}
 
 
364
  for dirpath, _dirs, fnames in os.walk(root):
365
- # Skip hidden dirs and node_modules / __pycache__
366
  parts = os.path.relpath(dirpath, root).split(os.sep)
367
  if any(p.startswith(".") or p in {"node_modules", "__pycache__", ".venv", "venv"} for p in parts):
368
  continue
@@ -375,4 +618,105 @@ def snapshot_workspace() -> dict[str, str]:
375
  files[os.path.relpath(full, root)] = f.read()
376
  except (UnicodeDecodeError, PermissionError):
377
  continue
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
378
  return files
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  """File system tools: read, write, edit, list, glob, grep.
2
 
3
+ Enhanced with Hugging Face Hub cloud storage integration:
4
+ - All file operations work seamlessly with both local and cloud storage
5
+ - Automatic sync to HF Hub when configured
6
+ - Transparent fallback between local and remote
7
+
8
  All tools are sandboxed to a configurable workspace root (default: ./workspace).
9
  They return JSON-serializable dicts so they can be exposed via the API.
10
  """
 
12
  from __future__ import annotations
13
 
14
  import fnmatch
15
+ import logging
16
  import os
17
  import re
18
  import shutil
19
  from pathlib import Path
20
+ from typing import Any, Optional
21
+
22
+ logger = logging.getLogger(__name__)
23
 
24
  # ─── Workspace sandbox ──────────────────────────────────────────────────
25
 
 
29
  os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "workspace")),
30
  )
31
 
32
+ # Hub storage instance (lazy initialized)
33
+ _hub_storage = None
34
+
35
 
36
  def get_workspace_root() -> str:
37
  """Return the absolute path of the agent's workspace root."""
 
60
  return full
61
 
62
 
63
+ def _get_hub_storage():
64
+ """Get the HubStorage instance if available."""
65
+ global _hub_storage
66
+ if _hub_storage is None:
67
+ try:
68
+ from code.hub_storage import get_hub_storage, is_hub_available
69
+ if is_hub_available():
70
+ _hub_storage = get_hub_storage()
71
+ logger.debug("Hub storage integration active")
72
+ else:
73
+ logger.debug("Hub storage not configured")
74
+ except ImportError:
75
+ logger.debug("huggingface_hub not installed")
76
+ _hub_storage = None
77
+ return _hub_storage
78
+
79
+
80
  # ─── read_file ──────────────────────────────────────────────────────────
81
 
82
  def read_file(path: str, offset: int = 0, limit: int | None = None) -> dict[str, Any]:
83
  """Read a text file from the workspace.
84
 
85
+ Enhanced: Falls back to Hub storage if file not found locally.
86
+
87
  Args:
88
  path: Relative path inside the workspace, or absolute within it.
89
  offset: 1-indexed line to start reading from.
 
94
  """
95
  try:
96
  full = _resolve_safe(path)
97
+
98
+ # Try local first
99
+ if os.path.exists(full) and os.path.isfile(full):
100
+ with open(full, "r", encoding="utf-8", errors="replace") as f:
101
+ lines = f.readlines()
102
+
103
+ total = len(lines)
104
+ start = max(0, (offset - 1) if offset > 0 else 0)
105
+ end = (start + limit) if limit else total
106
+ selected = lines[start:end]
107
+
108
+ # Re-number for display
109
+ numbered = "".join(
110
+ f"{start + i + 1:6}\t{line}" for i, line in enumerate(selected)
111
+ )
 
 
112
 
113
+ return {
114
+ "success": True,
115
+ "path": path,
116
+ "content": numbered,
117
+ "line_count": total,
118
+ "returned_lines": len(selected),
119
+ "truncated": end < total,
120
+ "source": "local",
121
+ }
122
+
123
+ # Try Hub storage
124
+ hub = _get_hub_storage()
125
+ if hub and hub.file_exists(path):
126
+ try:
127
+ content = hub.read_file(path)
128
+ lines = content.split('\n')
129
+
130
+ total = len(lines)
131
+ start = max(0, (offset - 1) if offset > 0 else 0)
132
+ end = (start + limit) if limit else total
133
+ selected = lines[start:end]
134
+
135
+ numbered = "".join(
136
+ f"{start + i + 1:6}\t{line}" for i, line in enumerate(selected)
137
+ )
138
+
139
+ return {
140
+ "success": True,
141
+ "path": path,
142
+ "content": numbered,
143
+ "line_count": total,
144
+ "returned_lines": len(selected),
145
+ "truncated": end < total,
146
+ "source": "hub",
147
+ }
148
+ except Exception as e:
149
+ logger.warning("Failed to read %s from Hub: %s", path, e)
150
+
151
+ return {"success": False, "error": f"File not found: {path}"}
152
+
153
  except Exception as exc:
154
  return {"success": False, "error": str(exc)}
155
 
 
157
  # ─── write_file ─────────────────────────────────────────────────────────
158
 
159
  def write_file(path: str, content: str) -> dict[str, Any]:
160
+ """Write content to a file, creating parent directories as needed.
161
+
162
+ Enhanced: Automatically syncs to HF Hub when configured.
163
+
164
+ Args:
165
+ path: Relative file path in workspace
166
+ content: File content string
167
+
168
+ Returns:
169
+ dict with success status and metadata
170
+ """
171
  try:
172
  full = _resolve_safe(path)
173
  os.makedirs(os.path.dirname(full), exist_ok=True)
174
+
175
+ # Write locally first
176
  with open(full, "w", encoding="utf-8") as f:
177
  f.write(content)
178
+
179
+ result = {
180
  "success": True,
181
  "path": path,
182
  "bytes_written": len(content.encode("utf-8")),
183
+ "source": "local",
184
  }
185
+
186
+ # Sync to Hub if configured
187
+ hub = _get_hub_storage()
188
+ if hub:
189
+ try:
190
+ metadata = hub.write_file(
191
+ path=path,
192
+ content=content,
193
+ commit_message=f"AI generated/updated: {path}"
194
+ )
195
+ result["hub_sync"] = True
196
+ result["hub_remote"] = metadata.is_remote
197
+ result["source"] = "synced"
198
+ logger.info("Synced %s to Hub", path)
199
+ except Exception as e:
200
+ logger.warning("Failed to sync %s to Hub: %s", path, e)
201
+ result["hub_sync"] = False
202
+ result["hub_error"] = str(e)
203
+
204
+ return result
205
+
206
  except Exception as exc:
207
  return {"success": False, "error": str(exc)}
208
 
 
215
  new_str: str,
216
  replace_all: bool = False,
217
  ) -> dict[str, Any]:
218
+ """Replace occurrences of old_str with new_str in a file.
219
+
220
+ Enhanced: Syncs changes back to Hub after edit.
221
+ """
222
  try:
223
  full = _resolve_safe(path)
224
+
225
+ # Read file (local or Hub)
226
+ hub = _get_hub_storage()
227
+ if os.path.exists(full):
228
+ with open(full, "r", encoding="utf-8") as f:
229
+ content = f.read()
230
+ elif hub and hub.file_exists(path):
231
+ content = hub.read_file(path)
232
+ else:
233
  return {"success": False, "error": f"File not found: {path}"}
234
 
 
 
 
235
  if old_str not in content:
236
  return {
237
  "success": False,
 
255
  old_str, new_str, 1
256
  )
257
 
258
+ # Write locally
259
+ os.makedirs(os.path.dirname(full), exist_ok=True)
260
  with open(full, "w", encoding="utf-8") as f:
261
  f.write(new_content)
262
 
263
+ result = {
264
  "success": True,
265
  "path": path,
266
  "replacements": count,
267
  }
268
+
269
+ # Sync to Hub
270
+ if hub:
271
+ try:
272
+ hub.write_file(path, new_content, commit_message=f"Edit: {path}")
273
+ result["hub_sync"] = True
274
+ except Exception as e:
275
+ logger.warning("Failed to sync edit to Hub: %s", e)
276
+
277
+ return result
278
+
279
  except Exception as exc:
280
  return {"success": False, "error": str(exc)}
281
 
282
 
283
  def multi_edit(path: str, edits: list[dict[str, Any]]) -> dict[str, Any]:
284
+ """Apply multiple edits to a file atomically (all-or-nothing).
285
+
286
+ Enhanced: Uses batch operation for Hub sync.
287
+ """
288
  try:
289
  full = _resolve_safe(path)
290
+
291
+ # Read file
292
+ hub = _get_hub_storage()
293
+ if os.path.exists(full):
294
+ with open(full, "r", encoding="utf-8") as f:
295
+ content = f.read()
296
+ elif hub and hub.file_exists(path):
297
+ content = hub.read_file(path)
298
+ else:
299
  return {"success": False, "error": f"File not found: {path}"}
300
 
 
 
 
301
  applied = 0
302
  for edit in edits:
303
  old_str = edit.get("old_str", "")
 
320
  )
321
  applied += 1
322
 
323
+ # Write locally
324
+ os.makedirs(os.path.dirname(full), exist_ok=True)
325
  with open(full, "w", encoding="utf-8") as f:
326
  f.write(content)
327
 
328
+ result = {"success": True, "path": path, "applied": applied}
329
+
330
+ # Sync to Hub
331
+ if hub:
332
+ try:
333
+ hub.write_file(path, content, commit_message=f"Multi-edit: {path}")
334
+ result["hub_sync"] = True
335
+ except Exception as e:
336
+ logger.warning("Failed to sync multi-edit to Hub: %s", e)
337
+
338
+ return result
339
+
340
  except Exception as exc:
341
  return {"success": False, "error": str(exc)}
342
 
 
344
  # ─── list_dir ───────────────────────────────────────────────────────────
345
 
346
  def list_dir(path: str = ".") -> dict[str, Any]:
347
+ """List directory contents.
348
+
349
+ Enhanced: Merges local and Hub files.
350
+ """
351
  try:
352
  full = _resolve_safe(path)
353
+
 
 
 
 
354
  entries = []
355
+ seen_names = set()
356
+
357
+ # List local files
358
+ if os.path.exists(full) and os.path.isdir(full):
359
+ for name in sorted(os.listdir(full)):
360
+ entry_path = os.path.join(full, name)
361
+ stat = os.stat(entry_path)
362
+ rel_path = os.path.relpath(entry_path, get_workspace_root())
363
+ entries.append({
364
+ "name": name,
365
+ "type": "dir" if os.path.isdir(entry_path) else "file",
366
+ "size": stat.st_size,
367
+ "path": rel_path.replace(os.sep, "/"),
368
+ "source": "local",
369
+ })
370
+ seen_names.add(name)
371
+
372
+ # Merge with Hub files
373
+ hub = _get_hub_storage()
374
+ if hub:
375
+ try:
376
+ hub_files = hub.list_files(path=path, recursive=False)
377
+ for hf in hub_files:
378
+ hf_name = Path(hf["path"]).name
379
+ if hf_name not in seen_names:
380
+ entries.append({
381
+ "name": hf_name,
382
+ "type": "file",
383
+ "size": hf.get("size", 0),
384
+ "path": hf["path"],
385
+ "source": "hub",
386
+ })
387
+ seen_names.add(hf_name)
388
+ except Exception as e:
389
+ logger.warning("Failed to list Hub files: %s", e)
390
 
391
  return {
392
  "success": True,
393
  "path": path,
394
+ "entries": sorted(entries, key=lambda x: x["name"]),
395
  }
396
+
397
  except Exception as exc:
398
  return {"success": False, "error": str(exc)}
399
 
 
401
  # ─── glob ───────────────────────────────────────────────────────────────
402
 
403
  def glob_paths(pattern: str, path: str = ".") -> dict[str, Any]:
404
+ """Glob file paths matching a pattern, recursively.
405
+
406
+ Enhanced: Searches both local and Hub storage.
407
+ """
408
  try:
409
  full = _resolve_safe(path)
410
+ matches: list[dict[str, Any]] = []
411
+ seen_paths = set()
412
+
413
+ # Local glob
414
+ if os.path.exists(full):
415
+ for root_dir, _dirs, files in os.walk(full):
416
+ for fname in files:
417
+ if fnmatch.fnmatch(fname, pattern) or fnmatch.fnmatch(
418
+ os.path.relpath(os.path.join(root_dir, fname), full), pattern
419
+ ):
420
+ rel_path = os.path.relpath(os.path.join(root_dir, fname), get_workspace_root()).replace(os.sep, "/")
421
+ if rel_path not in seen_paths:
422
+ matches.append({"path": rel_path, "source": "local"})
423
+ seen_paths.add(rel_path)
424
+
425
+ # Hub glob
426
+ hub = _get_hub_storage()
427
+ if hub:
428
+ try:
429
+ hub_files = hub.list_files(path=path, recursive=True)
430
+ for hf in hub_files:
431
+ if fnmatch.fnmatch(Path(hf["path"]).name, pattern) or fnmatch.fnmatch(hf["path"], pattern):
432
+ if hf["path"] not in seen_paths:
433
+ matches.append({"path": hf["path"], "source": "hub"})
434
+ seen_paths.add(hf["path"])
435
+ except Exception as e:
436
+ logger.warning("Failed to glob Hub files: %s", e)
437
+
438
+ matches.sort(key=lambda x: x["path"])
439
  return {"success": True, "pattern": pattern, "matches": matches}
440
+
441
  except Exception as exc:
442
  return {"success": False, "error": str(exc)}
443
 
 
451
  ignore_case: bool = False,
452
  max_results: int = 100,
453
  ) -> dict[str, Any]:
454
+ """Search file contents with a regex pattern.
455
+
456
+ Enhanced: Searches both local and Hub files.
457
+ """
458
  try:
 
459
  flags = re.IGNORECASE if ignore_case else 0
460
  regex = re.compile(pattern, flags)
461
 
462
  matches: list[dict[str, Any]] = []
463
+
464
+ # Search local files
465
+ full = _resolve_safe(path)
466
+ if os.path.exists(full):
467
+ for root_dir, _dirs, files in os.walk(full):
468
+ for fname in files:
469
+ if include and not fnmatch.fnmatch(fname, include):
470
+ continue
471
+ fpath = os.path.join(root_dir, fname)
472
+ try:
473
+ with open(fpath, "r", encoding="utf-8", errors="replace") as f:
474
+ for lineno, line in enumerate(f, 1):
475
+ if regex.search(line):
476
+ matches.append({
477
+ "file": os.path.relpath(fpath, get_workspace_root()).replace(os.sep, "/"),
478
+ "line": lineno,
479
+ "text": line.rstrip()[:500],
480
+ "source": "local",
481
+ })
482
+ if len(matches) >= max_results:
483
+ return {
484
+ "success": True,
485
+ "pattern": pattern,
486
+ "matches": matches,
487
+ "truncated": True,
488
+ }
489
+ except (UnicodeDecodeError, PermissionError):
490
+ continue
491
+
492
+ # Search Hub files
493
+ hub = _get_hub_storage()
494
+ if hub and len(matches) < max_results:
495
+ try:
496
+ hub_files = hub.list_files(path=path, recursive=True)
497
+ for hf in hub_files:
498
+ if include and not fnmatch.fnmatch(Path(hf["path"]).name, include):
499
+ continue
500
+ try:
501
+ content = hub.read_file(hf["path"])
502
+ for lineno, line in enumerate(content.split('\n'), 1):
503
  if regex.search(line):
504
  matches.append({
505
+ "file": hf["path"],
506
  "line": lineno,
507
  "text": line.rstrip()[:500],
508
+ "source": "hub",
509
  })
510
  if len(matches) >= max_results:
511
  return {
 
514
  "matches": matches,
515
  "truncated": True,
516
  }
517
+ except Exception:
518
+ continue
519
+ except Exception as e:
520
+ logger.warning("Failed to search Hub files: %s", e)
521
 
522
  return {
523
  "success": True,
 
525
  "matches": matches,
526
  "truncated": False,
527
  }
528
+
529
  except re.error as exc:
530
  return {"success": False, "error": f"Invalid regex: {exc}"}
531
  except Exception as exc:
 
535
  # ─── Workspace management ───────────────────────────────────────────────
536
 
537
  def list_workspace_tree(max_depth: int = 3) -> dict[str, Any]:
538
+ """Return a tree view of the workspace.
539
+
540
+ Enhanced: Includes Hub files.
541
+ """
542
  try:
543
  root = get_workspace_root()
544
 
 
562
  return {"name": os.path.basename(path), "type": "dir", "children": entries}
563
 
564
  tree = _walk(root, 0)
565
+
566
+ # Add Hub info
567
+ hub = _get_hub_storage()
568
+ if hub:
569
+ stats = hub.get_storage_stats()
570
+ tree["hub_enabled"] = stats["hub_enabled"]
571
+ tree["storage_mode"] = stats["mode"]
572
+
573
  return {"success": True, "tree": tree}
574
+
575
  except Exception as exc:
576
  return {"success": False, "error": str(exc)}
577
 
578
 
579
  def reset_workspace() -> dict[str, Any]:
580
+ """Clear all files in the workspace (used by /new command).
581
+
582
+ Note: Does NOT clear Hub storage - use reset_hub_workspace for that.
583
+ """
584
  try:
585
  root = get_workspace_root()
586
  if os.path.exists(root):
 
598
  def snapshot_workspace() -> dict[str, str]:
599
  """Return a dict of {relative_path: content} for all text files in the workspace.
600
 
601
+ Enhanced: Includes Hub files in snapshot.
602
  Used to package workspace files for ZIP/HF deploy.
603
  """
604
  root = get_workspace_root()
605
  files: dict[str, str] = {}
606
+
607
+ # Local files
608
  for dirpath, _dirs, fnames in os.walk(root):
 
609
  parts = os.path.relpath(dirpath, root).split(os.sep)
610
  if any(p.startswith(".") or p in {"node_modules", "__pycache__", ".venv", "venv"} for p in parts):
611
  continue
 
618
  files[os.path.relpath(full, root)] = f.read()
619
  except (UnicodeDecodeError, PermissionError):
620
  continue
621
+
622
+ # Hub files (merge, don't overwrite local)
623
+ hub = _get_hub_storage()
624
+ if hub:
625
+ try:
626
+ hub_files = hub.list_files()
627
+ for hf in hub_files:
628
+ if hf["path"] not in files:
629
+ try:
630
+ content = hub.read_file(hf["path"])
631
+ files[hf["path"]] = content
632
+ except Exception:
633
+ continue
634
+ except Exception as e:
635
+ logger.warning("Failed to snapshot Hub files: %s", e)
636
+
637
  return files
638
+
639
+
640
+ # ─── Hub-specific operations ────────────────────────────────────────────
641
+
642
+ def sync_to_hub(path: str | None = None) -> dict[str, Any]:
643
+ """Manually trigger sync to HF Hub.
644
+
645
+ Args:
646
+ path: Specific file/directory to sync (None = all)
647
+
648
+ Returns:
649
+ Sync result dict
650
+ """
651
+ hub = _get_hub_storage()
652
+ if not hub:
653
+ return {"success": False, "error": "Hub storage not configured"}
654
+
655
+ try:
656
+ return hub.sync_to_hub(path=path)
657
+ except Exception as e:
658
+ return {"success": False, "error": str(e)}
659
+
660
+
661
+ def sync_from_hub(path: str | None = None) -> dict[str, Any]:
662
+ """Manually trigger sync from HF Hub.
663
+
664
+ Args:
665
+ path: Specific file to download (None = all)
666
+
667
+ Returns:
668
+ Sync result dict
669
+ """
670
+ hub = _get_hub_storage()
671
+ if not hub:
672
+ return {"success": False, "error": "Hub storage not configured"}
673
+
674
+ try:
675
+ return hub.sync_from_hub(path=path)
676
+ except Exception as e:
677
+ return {"success": False, "error": str(e)}
678
+
679
+
680
+ def get_hub_status() -> dict[str, Any]:
681
+ """Get current Hub storage status and configuration."""
682
+ hub = _get_hub_storage()
683
+ if not hub:
684
+ return {
685
+ "available": False,
686
+ "configured": False,
687
+ "mode": "local_only",
688
+ }
689
+
690
+ try:
691
+ stats = hub.get_storage_stats()
692
+ return {
693
+ "available": True,
694
+ "configured": True,
695
+ **stats,
696
+ }
697
+ except Exception as e:
698
+ return {
699
+ "available": False,
700
+ "configured": True,
701
+ "error": str(e),
702
+ }
703
+
704
+
705
+ def delete_from_hub(path: str) -> dict[str, Any]:
706
+ """Delete a file from Hub storage only (keeps local).
707
+
708
+ Args:
709
+ path: File path to delete from Hub
710
+
711
+ Returns:
712
+ Operation result
713
+ """
714
+ hub = _get_hub_storage()
715
+ if not hub:
716
+ return {"success": False, "error": "Hub storage not configured"}
717
+
718
+ try:
719
+ success = hub.delete_file(path, commit_message=f"Delete: {path}")
720
+ return {"success": success, "path": path}
721
+ except Exception as e:
722
+ return {"success": False, "error": str(e)}