Spaces:
Running
Running
File size: 3,775 Bytes
fa2755b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 | import random
import os
import sys
import json
import csv
from huggingface_hub import hf_hub_download, HfApi
# Force UTF-8 for console output (Windows fix)
sys.stdout.reconfigure(encoding='utf-8')
# Add backend directory to sys.path so imports work
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from config import AppConfig
def main():
repo_id = "toecm/PureChain_Dataset"
filename = "pending_tx_queue.json"
queue = []
from huggingface_hub import get_token
hf_token = getattr(AppConfig, 'HF_TOKEN', None) or get_token()
if not hf_token:
print("❌ Missing HF_TOKEN in config and Hugging Face CLI.")
return
print("☁️ Downloading existing pending_tx_queue.json from Hugging Face...")
try:
downloaded_path = hf_hub_download(
repo_id=repo_id,
repo_type="dataset",
filename=filename,
token=hf_token
)
with open(downloaded_path, "r", encoding='utf-8') as f:
queue = json.load(f)
print(f"✅ Downloaded existing queue with {len(queue)} items.")
except Exception as e:
print(f"ℹ️ Could not download existing queue (it may not exist). Starting fresh. ({e})")
queue = []
korean_dialects = [
"Gyeongsangdo Satoori",
"Jeju Satoori",
"Gangwon Satoori",
"Chungcheong Satoori",
"Seoul Satoori",
"Jeolla Satoori",
]
print("🎲 Generating 60 Korean↔Korean interactions...")
for i in range(60):
d1 = random.choice(korean_dialects)
d2 = random.choice(korean_dialects)
payload = {
"original": f"Simulated utterance from {d1} to {d2}",
"dialect": d2,
"Data_Origin": f"Simulation: {d1}",
"user": "System Auto-Simulation"
}
queue.append(payload)
print("🎲 Generating 30 interactions for each Korean Satoori with American English...")
for dialect in korean_dialects:
for i in range(30):
payload = {
"original": f"Simulated utterance from {dialect} to American English",
"dialect": "American English",
"Data_Origin": f"Simulation: {dialect}",
"user": "System Auto-Simulation"
}
queue.append(payload)
print(f"✅ Generated simulations. Total queue size: {len(queue)}")
local_path = "pending_tx_queue.json"
with open(local_path, "w", encoding="utf-8") as f:
json.dump(queue, f, indent=2, ensure_ascii=False)
# Write CSV version for pending approvals
csv_path = "pending_approvals.csv"
with open(csv_path, "w", newline="", encoding="utf-8") as csvfile:
fieldnames = ["original", "dialect", "Data_Origin", "user"]
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
for item in queue:
writer.writerow({k: item.get(k, "") for k in fieldnames})
print("☁️ Uploading updated queue back to Hugging Face...")
api = HfApi(token=hf_token)
# Upload JSON queue
api.upload_file(
path_or_fileobj=local_path,
path_in_repo=filename,
repo_id=repo_id,
repo_type="dataset",
commit_message="🤖 Auto-queued 240 simulated interactions (JSON)"
)
# Upload CSV approvals
api.upload_file(
path_or_fileobj=csv_path,
path_in_repo="pending_approvals.csv",
repo_id=repo_id,
repo_type="dataset",
commit_message="🤖 Auto-queued 240 simulated interactions (CSV)"
)
print("✅ Successfully uploaded JSON and CSV! The UI should see the new pending items shortly.")
if __name__ == "__main__":
main()
|