Spaces:
Sleeping
Sleeping
File size: 20,327 Bytes
2930dae ceec48c 2930dae ceec48c 2930dae 5b64237 ceec48c 2930dae | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 | """
SupportEnv β Customer Support Ticket Triage data.
Task 1 (easy) β Ticket Classification
Task 2 (medium) β Information Extraction
Task 3 (hard) β Resolution Generation
"""
from __future__ import annotations
from typing import Any, Dict, List
# ---------------------------------------------------------------------------
# TASK 1 β Ticket Classification
# Agent must choose: category + priority
# Categories: billing | technical | account | feature_request | complaint | general
# Priorities: low | medium | high | critical
# ---------------------------------------------------------------------------
TASK1_TICKETS: List[Dict[str, Any]] = [
{
"ticket_id": "T1-001",
"subject": "Double-charged on my Pro subscription this month",
"body": (
"Hi, I noticed I was billed $49.99 twice on my credit card "
"on March 3rd for my Pro subscription. My account email is "
"alice@example.com. This is really frustrating β please issue "
"a refund for the duplicate charge as soon as possible."
),
"customer_tier": "pro",
"account_age_days": 420,
"previous_tickets": 0,
"attachments": [],
"ground_truth": {
"category": "billing",
"priority": "high",
},
},
{
"ticket_id": "T1-002",
"subject": "API rate limit keeps hitting 429 even though we're Enterprise",
"body": (
"Our production service has been receiving HTTP 429 errors from "
"your API for the past 2 hours. We're on the Enterprise plan which "
"should give us 10 000 req/min. Current usage is ~3 000 req/min "
"so we're well within limits. This is causing a production outage "
"for our customers. Ticket urgency: CRITICAL."
),
"customer_tier": "enterprise",
"account_age_days": 730,
"previous_tickets": 3,
"attachments": ["rate_limit_screenshot.png"],
"ground_truth": {
"category": "technical",
"priority": "critical",
},
},
{
"ticket_id": "T1-003",
"subject": "Can I change my account email address?",
"body": (
"Hello, I recently changed jobs and want to update the email "
"associated with my account from oldname@corp.com to "
"newname@newcorp.com. Can you walk me through how to do this? "
"There's no rush β just want to get it sorted at some point."
),
"customer_tier": "free",
"account_age_days": 60,
"previous_tickets": 0,
"attachments": [],
"ground_truth": {
"category": "account",
"priority": "low",
},
},
{
"ticket_id": "T1-004",
"subject": "Would love a dark mode option in the dashboard",
"body": (
"Hi team, long-time Pro user here. I spend a lot of time in the "
"dashboard and it would be fantastic if you could add a dark mode "
"toggle. Many other SaaS tools have this now and it really reduces "
"eye strain. Would be great to see this in a future release!"
),
"customer_tier": "pro",
"account_age_days": 900,
"previous_tickets": 1,
"attachments": [],
"ground_truth": {
"category": "feature_request",
"priority": "low",
},
},
{
"ticket_id": "T1-005",
"subject": "Your customer service is absolutely terrible",
"body": (
"I submitted a ticket three weeks ago about a bug in your export "
"feature and nobody has responded. This is completely unacceptable. "
"I am paying for a Pro subscription and I expect timely support. "
"If this is not resolved in 48 hours I will be requesting a full "
"refund and posting a review online."
),
"customer_tier": "pro",
"account_age_days": 200,
"previous_tickets": 2,
"attachments": [],
"ground_truth": {
"category": "complaint",
"priority": "high",
},
},
]
# ---------------------------------------------------------------------------
# TASK 2 β Information Extraction
# Agent must populate extracted_entities and required_actions.
# Ground truth defines exact entity keys and accepted values.
# ---------------------------------------------------------------------------
TASK2_TICKETS: List[Dict[str, Any]] = [
{
"ticket_id": "T2-001",
"subject": "Refund request β incorrect charge on invoice #INV-20240312",
"body": (
"Dear Support, I was incorrectly charged $199.00 on invoice "
"#INV-20240312 dated 2024-03-12. My account ID is ACC-78234. "
"The charge should have been $99.00 as per our agreed annual plan. "
"Please issue a refund of $100.00 to my card on file and send a "
"corrected invoice. My name is Robert Chen."
),
"customer_tier": "pro",
"account_age_days": 540,
"previous_tickets": 1,
"attachments": ["invoice_INV-20240312.pdf"],
"ground_truth": {
"entities": {
"customer_name": "Robert Chen",
"account_id": "ACC-78234",
"invoice_number": "INV-20240312",
"incorrect_amount": "199.00",
"correct_amount": "99.00",
"refund_amount": "100.00",
},
"required_actions": [
"issue_refund",
"send_corrected_invoice",
],
},
},
{
"ticket_id": "T2-002",
"subject": "SSO login broken after company domain migration",
"body": (
"Hi, our company just migrated from acme-old.com to acme-new.com. "
"Since the migration last Tuesday (2024-03-19), our 45 users cannot "
"log in via SSO. The error message is: 'SAML assertion domain "
"mismatch'. Our org ID is ORG-5512. We need this fixed ASAP as "
"nobody can access the platform. Contact: Maria Gonzalez, "
"IT Director."
),
"customer_tier": "enterprise",
"account_age_days": 1100,
"previous_tickets": 5,
"attachments": [],
"ground_truth": {
"entities": {
"contact_name": "Maria Gonzalez",
"contact_role": "IT Director",
"org_id": "ORG-5512",
"old_domain": "acme-old.com",
"new_domain": "acme-new.com",
"error_message": "SAML assertion domain mismatch",
"affected_users": "45",
},
"required_actions": [
"update_sso_domain_config",
"verify_saml_settings",
"notify_affected_users",
],
},
},
{
"ticket_id": "T2-003",
"subject": "Need to add 3 seats to our Team plan before end of quarter",
"body": (
"Hello, I'm the account admin for Brightfield Analytics "
"(account: ACC-11099). We currently have 12 seats on Team plan "
"and need to add 3 more seats before March 31st (end of our fiscal "
"quarter). The 3 new team members are: "
"dev1@brightfield.io, dev2@brightfield.io, dev3@brightfield.io. "
"Please prorate the cost. β James Park, VP Engineering"
),
"customer_tier": "pro",
"account_age_days": 820,
"previous_tickets": 3,
"attachments": [],
"ground_truth": {
"entities": {
"contact_name": "James Park",
"contact_role": "VP Engineering",
"account_id": "ACC-11099",
"company_name": "Brightfield Analytics",
"current_seats": "12",
"seats_to_add": "3",
"deadline": "March 31st",
"new_users": [
"dev1@brightfield.io",
"dev2@brightfield.io",
"dev3@brightfield.io",
],
},
"required_actions": [
"add_seats",
"send_prorated_invoice",
"provision_new_users",
],
},
},
{
"ticket_id": "T2-004",
"subject": "Data export stuck at 0% for 6 hours β export ID EXP-990021",
"body": (
"Our scheduled data export (ID: EXP-990021) has been stuck at 0% "
"for over 6 hours. This export contains 90 days of transaction "
"data for our compliance report due tomorrow. Account: ACC-30041, "
"region: eu-west-1. We need this export completed or the raw data "
"sent via secure link by 09:00 UTC tomorrow. If the export cannot "
"be fixed, please escalate to engineering. β Priya Sharma"
),
"customer_tier": "enterprise",
"account_age_days": 650,
"previous_tickets": 7,
"attachments": [],
"ground_truth": {
"entities": {
"contact_name": "Priya Sharma",
"export_id": "EXP-990021",
"account_id": "ACC-30041",
"region": "eu-west-1",
"data_range": "90 days",
"deadline": "09:00 UTC tomorrow",
},
"required_actions": [
"investigate_export_job",
"fix_or_restart_export",
"escalate_to_engineering",
],
},
},
{
"ticket_id": "T2-005",
"subject": "GDPR deletion request for former employee account",
"body": (
"We need to permanently delete all data associated with a former "
"employee's account as per GDPR Article 17 (right to erasure). "
"The account to delete: user ID USR-88821, email "
"j.doe@departed.co.uk. Our DPO is Claire Lambert "
"(dpo@ourenterprise.com). Legal reference: GDPR-REQ-2024-03. "
"Please confirm deletion in writing within 30 days. "
"Org ID: ORG-7740."
),
"customer_tier": "enterprise",
"account_age_days": 1400,
"previous_tickets": 2,
"attachments": ["gdpr_deletion_form.pdf"],
"ground_truth": {
"entities": {
"user_id": "USR-88821",
"user_email": "j.doe@departed.co.uk",
"org_id": "ORG-7740",
"dpo_name": "Claire Lambert",
"dpo_email": "dpo@ourenterprise.com",
"legal_reference": "GDPR-REQ-2024-03",
"legal_basis": "GDPR Article 17",
},
"required_actions": [
"delete_user_data",
"send_written_confirmation",
"log_gdpr_request",
],
},
},
]
# ---------------------------------------------------------------------------
# TASK 3 β Resolution Generation
# Agent must submit a response_text + resolution_steps.
# Ground truth contains required keywords and accepted resolution steps.
# Scoring is deterministic (keyword matching + step coverage).
# ---------------------------------------------------------------------------
TASK3_TICKETS: List[Dict[str, Any]] = [
{
"ticket_id": "T3-001",
"subject": "Cannot reset my password β reset email never arrives",
"body": (
"Hi, I've tried resetting my password 5 times today but the reset "
"email never arrives. I've checked my spam folder. My account email "
"is user@example.com. I need access to my account urgently as I "
"have a demo with a client in 2 hours."
),
"customer_tier": "pro",
"account_age_days": 280,
"previous_tickets": 0,
"attachments": [],
"ground_truth": {
"required_keywords": [
"password",
"reset",
"email",
"spam",
"whitelist",
],
"required_resolution_steps": [
"verify_email_delivery",
"check_spam_filters",
"manual_password_reset",
"follow_up_confirmation",
],
"tone_requirements": {
"must_apologize": True,
"must_acknowledge_urgency": True,
"must_provide_timeline": True,
},
"expected_response_length_min": 80,
},
},
{
"ticket_id": "T3-002",
"subject": "Webhook payloads stopped arriving after updating secret key",
"body": (
"We updated our webhook secret key yesterday in the dashboard and "
"now none of our webhook endpoints are receiving payloads. Our "
"endpoint is https://hooks.our-app.com/receive. We've verified the "
"endpoint is up and responding 200. It seems like the new secret "
"is not being used to sign payloads. Account: ACC-50039."
),
"customer_tier": "pro",
"account_age_days": 510,
"previous_tickets": 4,
"attachments": [],
"ground_truth": {
"required_keywords": [
"webhook",
"secret",
"signature",
"HMAC",
"regenerate",
],
"required_resolution_steps": [
"verify_webhook_config",
"regenerate_webhook_secret",
"update_endpoint_verification",
"test_delivery",
],
"tone_requirements": {
"must_apologize": False,
"must_acknowledge_urgency": False,
"must_provide_timeline": True,
},
"expected_response_length_min": 100,
},
},
{
"ticket_id": "T3-003",
"subject": "Annual invoice shows wrong VAT number β urgent for accounting",
"body": (
"Our annual invoice (INV-2024-ANN-00567) shows VAT number "
"DE123456789 but our correct VAT number is DE987654321. This "
"invoice needs to be corrected immediately as our accounting team "
"needs it for Q1 closing (deadline: end of today). "
"Company: MΓΌller GmbH, Account: ACC-20987."
),
"customer_tier": "enterprise",
"account_age_days": 1200,
"previous_tickets": 8,
"attachments": [],
"ground_truth": {
"required_keywords": [
"VAT",
"invoice",
"corrected",
"accounting",
"reissue",
],
"required_resolution_steps": [
"update_vat_number",
"reissue_invoice",
"send_corrected_invoice",
"confirm_receipt",
],
"tone_requirements": {
"must_apologize": True,
"must_acknowledge_urgency": True,
"must_provide_timeline": True,
},
"expected_response_length_min": 80,
},
},
{
"ticket_id": "T3-004",
"subject": "Team member accidentally deleted our production dataset",
"body": (
"One of our team members accidentally deleted dataset "
"DS-PROD-77. This dataset had 6 months of customer analytics data "
"that we haven't backed up externally. Is there any way to restore "
"it? We are on Enterprise plan. Org: ORG-3302. Please help "
"immediately β this data is critical for our quarterly review."
),
"customer_tier": "enterprise",
"account_age_days": 960,
"previous_tickets": 2,
"attachments": [],
"ground_truth": {
"required_keywords": [
"restore",
"backup",
"recovery",
"dataset",
"retention",
],
"required_resolution_steps": [
"check_retention_policy",
"attempt_data_recovery",
"escalate_to_engineering",
"provide_recovery_status",
],
"tone_requirements": {
"must_apologize": True,
"must_acknowledge_urgency": True,
"must_provide_timeline": True,
},
"expected_response_length_min": 100,
},
},
{
"ticket_id": "T3-005",
"subject": "Need formal SLA documentation for enterprise procurement",
"body": (
"Our procurement team is finalizing contracts and requires formal "
"SLA documentation for your Enterprise plan. Specifically we need: "
"uptime guarantee percentage, support response time commitments, "
"incident severity classification, and escalation procedures. "
"Contact: procurement@bigcorp.io. Org: ORG-8855."
),
"customer_tier": "enterprise",
"account_age_days": 30,
"previous_tickets": 0,
"attachments": [],
"ground_truth": {
"required_keywords": [
"SLA",
"uptime",
"response time",
"documentation",
"enterprise",
],
"required_resolution_steps": [
"provide_sla_document",
"highlight_enterprise_terms",
"connect_with_account_manager",
"confirm_receipt",
],
"tone_requirements": {
"must_apologize": False,
"must_acknowledge_urgency": False,
"must_provide_timeline": True,
},
"expected_response_length_min": 80,
},
},
]
# ---------------------------------------------------------------------------
# Task-level metadata
# ---------------------------------------------------------------------------
TASK_META: Dict[str, Dict[str, Any]] = {
"task1": {
"name": "Ticket Classification",
"description": (
"Classify each support ticket by category and priority. "
"Correct classification routes the ticket to the right team and "
"sets appropriate response SLAs."
),
"difficulty": "easy",
"max_steps": 3,
"tickets": TASK1_TICKETS,
"available_actions": ["classify", "submit"],
},
"task2": {
"name": "Information Extraction",
"description": (
"Extract structured entities (IDs, names, amounts, dates) from "
"tickets and identify the list of required actions needed to "
"resolve each case. Accuracy drives downstream automation."
),
"difficulty": "medium",
"max_steps": 5,
"tickets": TASK2_TICKETS,
"available_actions": ["extract", "respond", "submit"],
},
"task3": {
"name": "Resolution Generation",
"description": (
"Generate a complete, professional customer-facing response "
"plus an ordered list of resolution steps for each ticket. "
"Responses are graded on keyword coverage, step completeness, "
"tone adherence, and minimum length."
),
"difficulty": "hard",
"max_steps": 8,
"tickets": TASK3_TICKETS,
"available_actions": ["respond", "resolve", "escalate", "submit"],
},
}
def get_tickets(task_id: str) -> List[Dict[str, Any]]:
"""Return the ticket list for a given task."""
return TASK_META[task_id]["tickets"]
def get_task_meta(task_id: str) -> Dict[str, Any]:
"""Return task metadata (without ticket ground truth exposed to agent)."""
meta = dict(TASK_META[task_id])
# Strip ground_truth from tickets before returning to agents
safe_tickets = []
for t in meta["tickets"]:
safe_t = {k: v for k, v in t.items() if k != "ground_truth"}
safe_tickets.append(safe_t)
meta["tickets"] = safe_tickets
return meta |