Spaces:
Sleeping
Sleeping
| """ | |
| SupportEnv β Customer Support Ticket Triage data. | |
| Task 1 (easy) β Ticket Classification | |
| Task 2 (medium) β Information Extraction | |
| Task 3 (hard) β Resolution Generation | |
| """ | |
| from __future__ import annotations | |
| from typing import Any, Dict, List | |
| # --------------------------------------------------------------------------- | |
| # TASK 1 β Ticket Classification | |
| # Agent must choose: category + priority | |
| # Categories: billing | technical | account | feature_request | complaint | general | |
| # Priorities: low | medium | high | critical | |
| # --------------------------------------------------------------------------- | |
| TASK1_TICKETS: List[Dict[str, Any]] = [ | |
| { | |
| "ticket_id": "T1-001", | |
| "subject": "Double-charged on my Pro subscription this month", | |
| "body": ( | |
| "Hi, I noticed I was billed $49.99 twice on my credit card " | |
| "on March 3rd for my Pro subscription. My account email is " | |
| "alice@example.com. This is really frustrating β please issue " | |
| "a refund for the duplicate charge as soon as possible." | |
| ), | |
| "customer_tier": "pro", | |
| "account_age_days": 420, | |
| "previous_tickets": 0, | |
| "attachments": [], | |
| "ground_truth": { | |
| "category": "billing", | |
| "priority": "high", | |
| }, | |
| }, | |
| { | |
| "ticket_id": "T1-002", | |
| "subject": "API rate limit keeps hitting 429 even though we're Enterprise", | |
| "body": ( | |
| "Our production service has been receiving HTTP 429 errors from " | |
| "your API for the past 2 hours. We're on the Enterprise plan which " | |
| "should give us 10 000 req/min. Current usage is ~3 000 req/min " | |
| "so we're well within limits. This is causing a production outage " | |
| "for our customers. Ticket urgency: CRITICAL." | |
| ), | |
| "customer_tier": "enterprise", | |
| "account_age_days": 730, | |
| "previous_tickets": 3, | |
| "attachments": ["rate_limit_screenshot.png"], | |
| "ground_truth": { | |
| "category": "technical", | |
| "priority": "critical", | |
| }, | |
| }, | |
| { | |
| "ticket_id": "T1-003", | |
| "subject": "Can I change my account email address?", | |
| "body": ( | |
| "Hello, I recently changed jobs and want to update the email " | |
| "associated with my account from oldname@corp.com to " | |
| "newname@newcorp.com. Can you walk me through how to do this? " | |
| "There's no rush β just want to get it sorted at some point." | |
| ), | |
| "customer_tier": "free", | |
| "account_age_days": 60, | |
| "previous_tickets": 0, | |
| "attachments": [], | |
| "ground_truth": { | |
| "category": "account", | |
| "priority": "low", | |
| }, | |
| }, | |
| { | |
| "ticket_id": "T1-004", | |
| "subject": "Would love a dark mode option in the dashboard", | |
| "body": ( | |
| "Hi team, long-time Pro user here. I spend a lot of time in the " | |
| "dashboard and it would be fantastic if you could add a dark mode " | |
| "toggle. Many other SaaS tools have this now and it really reduces " | |
| "eye strain. Would be great to see this in a future release!" | |
| ), | |
| "customer_tier": "pro", | |
| "account_age_days": 900, | |
| "previous_tickets": 1, | |
| "attachments": [], | |
| "ground_truth": { | |
| "category": "feature_request", | |
| "priority": "low", | |
| }, | |
| }, | |
| { | |
| "ticket_id": "T1-005", | |
| "subject": "Your customer service is absolutely terrible", | |
| "body": ( | |
| "I submitted a ticket three weeks ago about a bug in your export " | |
| "feature and nobody has responded. This is completely unacceptable. " | |
| "I am paying for a Pro subscription and I expect timely support. " | |
| "If this is not resolved in 48 hours I will be requesting a full " | |
| "refund and posting a review online." | |
| ), | |
| "customer_tier": "pro", | |
| "account_age_days": 200, | |
| "previous_tickets": 2, | |
| "attachments": [], | |
| "ground_truth": { | |
| "category": "complaint", | |
| "priority": "high", | |
| }, | |
| }, | |
| ] | |
| # --------------------------------------------------------------------------- | |
| # TASK 2 β Information Extraction | |
| # Agent must populate extracted_entities and required_actions. | |
| # Ground truth defines exact entity keys and accepted values. | |
| # --------------------------------------------------------------------------- | |
| TASK2_TICKETS: List[Dict[str, Any]] = [ | |
| { | |
| "ticket_id": "T2-001", | |
| "subject": "Refund request β incorrect charge on invoice #INV-20240312", | |
| "body": ( | |
| "Dear Support, I was incorrectly charged $199.00 on invoice " | |
| "#INV-20240312 dated 2024-03-12. My account ID is ACC-78234. " | |
| "The charge should have been $99.00 as per our agreed annual plan. " | |
| "Please issue a refund of $100.00 to my card on file and send a " | |
| "corrected invoice. My name is Robert Chen." | |
| ), | |
| "customer_tier": "pro", | |
| "account_age_days": 540, | |
| "previous_tickets": 1, | |
| "attachments": ["invoice_INV-20240312.pdf"], | |
| "ground_truth": { | |
| "entities": { | |
| "customer_name": "Robert Chen", | |
| "account_id": "ACC-78234", | |
| "invoice_number": "INV-20240312", | |
| "incorrect_amount": "199.00", | |
| "correct_amount": "99.00", | |
| "refund_amount": "100.00", | |
| }, | |
| "required_actions": [ | |
| "issue_refund", | |
| "send_corrected_invoice", | |
| ], | |
| }, | |
| }, | |
| { | |
| "ticket_id": "T2-002", | |
| "subject": "SSO login broken after company domain migration", | |
| "body": ( | |
| "Hi, our company just migrated from acme-old.com to acme-new.com. " | |
| "Since the migration last Tuesday (2024-03-19), our 45 users cannot " | |
| "log in via SSO. The error message is: 'SAML assertion domain " | |
| "mismatch'. Our org ID is ORG-5512. We need this fixed ASAP as " | |
| "nobody can access the platform. Contact: Maria Gonzalez, " | |
| "IT Director." | |
| ), | |
| "customer_tier": "enterprise", | |
| "account_age_days": 1100, | |
| "previous_tickets": 5, | |
| "attachments": [], | |
| "ground_truth": { | |
| "entities": { | |
| "contact_name": "Maria Gonzalez", | |
| "contact_role": "IT Director", | |
| "org_id": "ORG-5512", | |
| "old_domain": "acme-old.com", | |
| "new_domain": "acme-new.com", | |
| "error_message": "SAML assertion domain mismatch", | |
| "affected_users": "45", | |
| }, | |
| "required_actions": [ | |
| "update_sso_domain_config", | |
| "verify_saml_settings", | |
| "notify_affected_users", | |
| ], | |
| }, | |
| }, | |
| { | |
| "ticket_id": "T2-003", | |
| "subject": "Need to add 3 seats to our Team plan before end of quarter", | |
| "body": ( | |
| "Hello, I'm the account admin for Brightfield Analytics " | |
| "(account: ACC-11099). We currently have 12 seats on Team plan " | |
| "and need to add 3 more seats before March 31st (end of our fiscal " | |
| "quarter). The 3 new team members are: " | |
| "dev1@brightfield.io, dev2@brightfield.io, dev3@brightfield.io. " | |
| "Please prorate the cost. β James Park, VP Engineering" | |
| ), | |
| "customer_tier": "pro", | |
| "account_age_days": 820, | |
| "previous_tickets": 3, | |
| "attachments": [], | |
| "ground_truth": { | |
| "entities": { | |
| "contact_name": "James Park", | |
| "contact_role": "VP Engineering", | |
| "account_id": "ACC-11099", | |
| "company_name": "Brightfield Analytics", | |
| "current_seats": "12", | |
| "seats_to_add": "3", | |
| "deadline": "March 31st", | |
| "new_users": [ | |
| "dev1@brightfield.io", | |
| "dev2@brightfield.io", | |
| "dev3@brightfield.io", | |
| ], | |
| }, | |
| "required_actions": [ | |
| "add_seats", | |
| "send_prorated_invoice", | |
| "provision_new_users", | |
| ], | |
| }, | |
| }, | |
| { | |
| "ticket_id": "T2-004", | |
| "subject": "Data export stuck at 0% for 6 hours β export ID EXP-990021", | |
| "body": ( | |
| "Our scheduled data export (ID: EXP-990021) has been stuck at 0% " | |
| "for over 6 hours. This export contains 90 days of transaction " | |
| "data for our compliance report due tomorrow. Account: ACC-30041, " | |
| "region: eu-west-1. We need this export completed or the raw data " | |
| "sent via secure link by 09:00 UTC tomorrow. If the export cannot " | |
| "be fixed, please escalate to engineering. β Priya Sharma" | |
| ), | |
| "customer_tier": "enterprise", | |
| "account_age_days": 650, | |
| "previous_tickets": 7, | |
| "attachments": [], | |
| "ground_truth": { | |
| "entities": { | |
| "contact_name": "Priya Sharma", | |
| "export_id": "EXP-990021", | |
| "account_id": "ACC-30041", | |
| "region": "eu-west-1", | |
| "data_range": "90 days", | |
| "deadline": "09:00 UTC tomorrow", | |
| }, | |
| "required_actions": [ | |
| "investigate_export_job", | |
| "fix_or_restart_export", | |
| "escalate_to_engineering", | |
| ], | |
| }, | |
| }, | |
| { | |
| "ticket_id": "T2-005", | |
| "subject": "GDPR deletion request for former employee account", | |
| "body": ( | |
| "We need to permanently delete all data associated with a former " | |
| "employee's account as per GDPR Article 17 (right to erasure). " | |
| "The account to delete: user ID USR-88821, email " | |
| "j.doe@departed.co.uk. Our DPO is Claire Lambert " | |
| "(dpo@ourenterprise.com). Legal reference: GDPR-REQ-2024-03. " | |
| "Please confirm deletion in writing within 30 days. " | |
| "Org ID: ORG-7740." | |
| ), | |
| "customer_tier": "enterprise", | |
| "account_age_days": 1400, | |
| "previous_tickets": 2, | |
| "attachments": ["gdpr_deletion_form.pdf"], | |
| "ground_truth": { | |
| "entities": { | |
| "user_id": "USR-88821", | |
| "user_email": "j.doe@departed.co.uk", | |
| "org_id": "ORG-7740", | |
| "dpo_name": "Claire Lambert", | |
| "dpo_email": "dpo@ourenterprise.com", | |
| "legal_reference": "GDPR-REQ-2024-03", | |
| "legal_basis": "GDPR Article 17", | |
| }, | |
| "required_actions": [ | |
| "delete_user_data", | |
| "send_written_confirmation", | |
| "log_gdpr_request", | |
| ], | |
| }, | |
| }, | |
| ] | |
| # --------------------------------------------------------------------------- | |
| # TASK 3 β Resolution Generation | |
| # Agent must submit a response_text + resolution_steps. | |
| # Ground truth contains required keywords and accepted resolution steps. | |
| # Scoring is deterministic (keyword matching + step coverage). | |
| # --------------------------------------------------------------------------- | |
| TASK3_TICKETS: List[Dict[str, Any]] = [ | |
| { | |
| "ticket_id": "T3-001", | |
| "subject": "Cannot reset my password β reset email never arrives", | |
| "body": ( | |
| "Hi, I've tried resetting my password 5 times today but the reset " | |
| "email never arrives. I've checked my spam folder. My account email " | |
| "is user@example.com. I need access to my account urgently as I " | |
| "have a demo with a client in 2 hours." | |
| ), | |
| "customer_tier": "pro", | |
| "account_age_days": 280, | |
| "previous_tickets": 0, | |
| "attachments": [], | |
| "ground_truth": { | |
| "required_keywords": [ | |
| "password", | |
| "reset", | |
| "email", | |
| "spam", | |
| "whitelist", | |
| ], | |
| "required_resolution_steps": [ | |
| "verify_email_delivery", | |
| "check_spam_filters", | |
| "manual_password_reset", | |
| "follow_up_confirmation", | |
| ], | |
| "tone_requirements": { | |
| "must_apologize": True, | |
| "must_acknowledge_urgency": True, | |
| "must_provide_timeline": True, | |
| }, | |
| "expected_response_length_min": 80, | |
| }, | |
| }, | |
| { | |
| "ticket_id": "T3-002", | |
| "subject": "Webhook payloads stopped arriving after updating secret key", | |
| "body": ( | |
| "We updated our webhook secret key yesterday in the dashboard and " | |
| "now none of our webhook endpoints are receiving payloads. Our " | |
| "endpoint is https://hooks.our-app.com/receive. We've verified the " | |
| "endpoint is up and responding 200. It seems like the new secret " | |
| "is not being used to sign payloads. Account: ACC-50039." | |
| ), | |
| "customer_tier": "pro", | |
| "account_age_days": 510, | |
| "previous_tickets": 4, | |
| "attachments": [], | |
| "ground_truth": { | |
| "required_keywords": [ | |
| "webhook", | |
| "secret", | |
| "signature", | |
| "HMAC", | |
| "regenerate", | |
| ], | |
| "required_resolution_steps": [ | |
| "verify_webhook_config", | |
| "regenerate_webhook_secret", | |
| "update_endpoint_verification", | |
| "test_delivery", | |
| ], | |
| "tone_requirements": { | |
| "must_apologize": False, | |
| "must_acknowledge_urgency": False, | |
| "must_provide_timeline": True, | |
| }, | |
| "expected_response_length_min": 100, | |
| }, | |
| }, | |
| { | |
| "ticket_id": "T3-003", | |
| "subject": "Annual invoice shows wrong VAT number β urgent for accounting", | |
| "body": ( | |
| "Our annual invoice (INV-2024-ANN-00567) shows VAT number " | |
| "DE123456789 but our correct VAT number is DE987654321. This " | |
| "invoice needs to be corrected immediately as our accounting team " | |
| "needs it for Q1 closing (deadline: end of today). " | |
| "Company: MΓΌller GmbH, Account: ACC-20987." | |
| ), | |
| "customer_tier": "enterprise", | |
| "account_age_days": 1200, | |
| "previous_tickets": 8, | |
| "attachments": [], | |
| "ground_truth": { | |
| "required_keywords": [ | |
| "VAT", | |
| "invoice", | |
| "corrected", | |
| "accounting", | |
| "reissue", | |
| ], | |
| "required_resolution_steps": [ | |
| "update_vat_number", | |
| "reissue_invoice", | |
| "send_corrected_invoice", | |
| "confirm_receipt", | |
| ], | |
| "tone_requirements": { | |
| "must_apologize": True, | |
| "must_acknowledge_urgency": True, | |
| "must_provide_timeline": True, | |
| }, | |
| "expected_response_length_min": 80, | |
| }, | |
| }, | |
| { | |
| "ticket_id": "T3-004", | |
| "subject": "Team member accidentally deleted our production dataset", | |
| "body": ( | |
| "One of our team members accidentally deleted dataset " | |
| "DS-PROD-77. This dataset had 6 months of customer analytics data " | |
| "that we haven't backed up externally. Is there any way to restore " | |
| "it? We are on Enterprise plan. Org: ORG-3302. Please help " | |
| "immediately β this data is critical for our quarterly review." | |
| ), | |
| "customer_tier": "enterprise", | |
| "account_age_days": 960, | |
| "previous_tickets": 2, | |
| "attachments": [], | |
| "ground_truth": { | |
| "required_keywords": [ | |
| "restore", | |
| "backup", | |
| "recovery", | |
| "dataset", | |
| "retention", | |
| ], | |
| "required_resolution_steps": [ | |
| "check_retention_policy", | |
| "attempt_data_recovery", | |
| "escalate_to_engineering", | |
| "provide_recovery_status", | |
| ], | |
| "tone_requirements": { | |
| "must_apologize": True, | |
| "must_acknowledge_urgency": True, | |
| "must_provide_timeline": True, | |
| }, | |
| "expected_response_length_min": 100, | |
| }, | |
| }, | |
| { | |
| "ticket_id": "T3-005", | |
| "subject": "Need formal SLA documentation for enterprise procurement", | |
| "body": ( | |
| "Our procurement team is finalizing contracts and requires formal " | |
| "SLA documentation for your Enterprise plan. Specifically we need: " | |
| "uptime guarantee percentage, support response time commitments, " | |
| "incident severity classification, and escalation procedures. " | |
| "Contact: procurement@bigcorp.io. Org: ORG-8855." | |
| ), | |
| "customer_tier": "enterprise", | |
| "account_age_days": 30, | |
| "previous_tickets": 0, | |
| "attachments": [], | |
| "ground_truth": { | |
| "required_keywords": [ | |
| "SLA", | |
| "uptime", | |
| "response time", | |
| "documentation", | |
| "enterprise", | |
| ], | |
| "required_resolution_steps": [ | |
| "provide_sla_document", | |
| "highlight_enterprise_terms", | |
| "connect_with_account_manager", | |
| "confirm_receipt", | |
| ], | |
| "tone_requirements": { | |
| "must_apologize": False, | |
| "must_acknowledge_urgency": False, | |
| "must_provide_timeline": True, | |
| }, | |
| "expected_response_length_min": 80, | |
| }, | |
| }, | |
| ] | |
| # --------------------------------------------------------------------------- | |
| # Task-level metadata | |
| # --------------------------------------------------------------------------- | |
| TASK_META: Dict[str, Dict[str, Any]] = { | |
| "task1": { | |
| "name": "Ticket Classification", | |
| "description": ( | |
| "Classify each support ticket by category and priority. " | |
| "Correct classification routes the ticket to the right team and " | |
| "sets appropriate response SLAs." | |
| ), | |
| "difficulty": "easy", | |
| "max_steps": 3, | |
| "tickets": TASK1_TICKETS, | |
| "available_actions": ["classify", "submit"], | |
| }, | |
| "task2": { | |
| "name": "Information Extraction", | |
| "description": ( | |
| "Extract structured entities (IDs, names, amounts, dates) from " | |
| "tickets and identify the list of required actions needed to " | |
| "resolve each case. Accuracy drives downstream automation." | |
| ), | |
| "difficulty": "medium", | |
| "max_steps": 5, | |
| "tickets": TASK2_TICKETS, | |
| "available_actions": ["extract", "respond", "submit"], | |
| }, | |
| "task3": { | |
| "name": "Resolution Generation", | |
| "description": ( | |
| "Generate a complete, professional customer-facing response " | |
| "plus an ordered list of resolution steps for each ticket. " | |
| "Responses are graded on keyword coverage, step completeness, " | |
| "tone adherence, and minimum length." | |
| ), | |
| "difficulty": "hard", | |
| "max_steps": 8, | |
| "tickets": TASK3_TICKETS, | |
| "available_actions": ["respond", "resolve", "escalate", "submit"], | |
| }, | |
| } | |
| def get_tickets(task_id: str) -> List[Dict[str, Any]]: | |
| """Return the ticket list for a given task.""" | |
| return TASK_META[task_id]["tickets"] | |
| def get_task_meta(task_id: str) -> Dict[str, Any]: | |
| """Return task metadata (without ticket ground truth exposed to agent).""" | |
| meta = dict(TASK_META[task_id]) | |
| # Strip ground_truth from tickets before returning to agents | |
| safe_tickets = [] | |
| for t in meta["tickets"]: | |
| safe_t = {k: v for k, v in t.items() if k != "ground_truth"} | |
| safe_tickets.append(safe_t) | |
| meta["tickets"] = safe_tickets | |
| return meta |