Spaces:
Running
Running
| import random | |
| import os | |
| import sys | |
| import json | |
| import csv | |
| from huggingface_hub import hf_hub_download, HfApi | |
| # Force UTF-8 for console output (Windows fix) | |
| sys.stdout.reconfigure(encoding='utf-8') | |
| # Add backend directory to sys.path so imports work | |
| sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) | |
| from config import AppConfig | |
| def main(): | |
| repo_id = "toecm/PureChain_Dataset" | |
| filename = "pending_tx_queue.json" | |
| queue = [] | |
| from huggingface_hub import get_token | |
| hf_token = getattr(AppConfig, 'HF_TOKEN', None) or get_token() | |
| if not hf_token: | |
| print("❌ Missing HF_TOKEN in config and Hugging Face CLI.") | |
| return | |
| print("☁️ Downloading existing pending_tx_queue.json from Hugging Face...") | |
| try: | |
| downloaded_path = hf_hub_download( | |
| repo_id=repo_id, | |
| repo_type="dataset", | |
| filename=filename, | |
| token=hf_token | |
| ) | |
| with open(downloaded_path, "r", encoding='utf-8') as f: | |
| queue = json.load(f) | |
| print(f"✅ Downloaded existing queue with {len(queue)} items.") | |
| except Exception as e: | |
| print(f"ℹ️ Could not download existing queue (it may not exist). Starting fresh. ({e})") | |
| queue = [] | |
| korean_dialects = [ | |
| "Gyeongsangdo Satoori", | |
| "Jeju Satoori", | |
| "Gangwon Satoori", | |
| "Chungcheong Satoori", | |
| "Seoul Satoori", | |
| "Jeolla Satoori", | |
| ] | |
| print("🎲 Generating 60 Korean↔Korean interactions...") | |
| for i in range(60): | |
| d1 = random.choice(korean_dialects) | |
| d2 = random.choice(korean_dialects) | |
| payload = { | |
| "original": f"Simulated utterance from {d1} to {d2}", | |
| "dialect": d2, | |
| "Data_Origin": f"Simulation: {d1}", | |
| "user": "System Auto-Simulation" | |
| } | |
| queue.append(payload) | |
| print("🎲 Generating 30 interactions for each Korean Satoori with American English...") | |
| for dialect in korean_dialects: | |
| for i in range(30): | |
| payload = { | |
| "original": f"Simulated utterance from {dialect} to American English", | |
| "dialect": "American English", | |
| "Data_Origin": f"Simulation: {dialect}", | |
| "user": "System Auto-Simulation" | |
| } | |
| queue.append(payload) | |
| print(f"✅ Generated simulations. Total queue size: {len(queue)}") | |
| local_path = "pending_tx_queue.json" | |
| with open(local_path, "w", encoding="utf-8") as f: | |
| json.dump(queue, f, indent=2, ensure_ascii=False) | |
| # Write CSV version for pending approvals | |
| csv_path = "pending_approvals.csv" | |
| with open(csv_path, "w", newline="", encoding="utf-8") as csvfile: | |
| fieldnames = ["original", "dialect", "Data_Origin", "user"] | |
| writer = csv.DictWriter(csvfile, fieldnames=fieldnames) | |
| writer.writeheader() | |
| for item in queue: | |
| writer.writerow({k: item.get(k, "") for k in fieldnames}) | |
| print("☁️ Uploading updated queue back to Hugging Face...") | |
| api = HfApi(token=hf_token) | |
| # Upload JSON queue | |
| api.upload_file( | |
| path_or_fileobj=local_path, | |
| path_in_repo=filename, | |
| repo_id=repo_id, | |
| repo_type="dataset", | |
| commit_message="🤖 Auto-queued 240 simulated interactions (JSON)" | |
| ) | |
| # Upload CSV approvals | |
| api.upload_file( | |
| path_or_fileobj=csv_path, | |
| path_in_repo="pending_approvals.csv", | |
| repo_id=repo_id, | |
| repo_type="dataset", | |
| commit_message="🤖 Auto-queued 240 simulated interactions (CSV)" | |
| ) | |
| print("✅ Successfully uploaded JSON and CSV! The UI should see the new pending items shortly.") | |
| if __name__ == "__main__": | |
| main() | |