File size: 4,152 Bytes
14d4369 dd04a48 14d4369 dd04a48 14d4369 a1efd54 14d4369 dd04a48 7b16ded dd04a48 14d4369 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 | import os
import time
import shutil
from pathlib import Path
from huggingface_hub import HfApi, snapshot_download
# Configuration fetched from Space Secrets
REPO_ID = os.getenv("DATASET_REPO_ID")
TOKEN = os.getenv("HF_TOKEN")
# Core directories
WORKSPACE_DIR = Path("/home/node/workspace")
OPENCODE_DIR = Path("/home/node/.local/share/opencode") # Default Linux storage
GLOBAL_CONFIG_DIR = Path("/home/node/.config/opencode") # Global AGENTS.md + skills
CACHE_DIR = Path("/home/node/.sync_cache")
INTERVAL = 30 * 60 # 30 minutes in seconds
api = HfApi(token=TOKEN)
def initial_pull():
if not REPO_ID:
print("No DATASET_REPO_ID provided. Skipping pull.")
return
print("Fetching dataset on startup...")
try:
# Download the latest snapshot of the private dataset
local_dir = snapshot_download(repo_id=REPO_ID, repo_type="dataset", token=TOKEN)
# Restore workspace files (includes env.example / env.json.local)
ds_workspace = Path(local_dir) / "workspace"
if ds_workspace.exists():
shutil.copytree(ds_workspace, WORKSPACE_DIR, dirs_exist_ok=True)
# Restore OpenCode sessions and history
ds_opencode = Path(local_dir) / "opencode"
if ds_opencode.exists():
shutil.copytree(ds_opencode, OPENCODE_DIR, dirs_exist_ok=True)
# Restore global config (AGENTS.md and skills)
ds_global = Path(local_dir) / "global"
if ds_global.exists():
shutil.copytree(ds_global, GLOBAL_CONFIG_DIR, dirs_exist_ok=True)
# Create an initial cache to track future deletions
shutil.copytree(OPENCODE_DIR, CACHE_DIR, dirs_exist_ok=True)
print("Restoration complete.")
except Exception as e:
print(f"Initial pull failed (likely an empty/new dataset): {e}")
def push_to_dataset():
if not REPO_ID:
return
print("Initiating 30-minute sync cycle...")
session_dir = OPENCODE_DIR / "storage" / "session"
cache_session_dir = CACHE_DIR / "storage" / "session"
# 1. Check for trashed sessions
if cache_session_dir.exists():
for cache_file in cache_session_dir.rglob("*.json"):
relative_path = cache_file.relative_to(cache_session_dir)
current_file = session_dir / relative_path
if not current_file.exists():
print(f"Detected deleted session: {relative_path}. Moving to trash...")
api.upload_file(
path_or_fileobj=str(cache_file),
path_in_repo=f"trashed/sessions/{relative_path}",
repo_id=REPO_ID,
repo_type="dataset",
token=TOKEN
)
# 2. Sync workspace and configuration
# The API automatically ignores the push if files are unchanged/empty.
# Files like env.json.local will upload securely since the dataset is private.
api.upload_folder(
folder_path=str(WORKSPACE_DIR),
path_in_repo="workspace",
repo_id=REPO_ID,
repo_type="dataset",
token=TOKEN,
ignore_patterns=["**/__pycache__/*", "**/.git/*"]
)
api.upload_folder(
folder_path=str(OPENCODE_DIR),
path_in_repo="opencode",
repo_id=REPO_ID,
repo_type="dataset",
token=TOKEN,
ignore_patterns=["**/*.log", "log", "log/**", "**/log/*"]
)
# 3. Sync global config (AGENTS.md and skills)
api.upload_folder(
folder_path=str(GLOBAL_CONFIG_DIR),
path_in_repo="global",
repo_id=REPO_ID,
repo_type="dataset",
token=TOKEN,
ignore_patterns=["**/node_modules/**", "node_modules/**", "node_modules", "**/*.log"]
)
# 4. Rebuild local deletion cache
if CACHE_DIR.exists():
shutil.rmtree(CACHE_DIR)
shutil.copytree(OPENCODE_DIR, CACHE_DIR, dirs_exist_ok=True)
print("Sync cycle finished successfully.")
if __name__ == "__main__":
initial_pull()
while True:
time.sleep(INTERVAL)
push_to_dataset()
|