data/benchmarks/conformity_ground_truth_benchmark.jsonl

{"id": "bench_001", "sentence": "A clinician-in-the-loop human oversight mechanism is strictly operational before diagnosis is confirmed.", "gold_category": "HUMAN_OVERSIGHT", "gold_status": "IMPLEMENTED", "target_article": "Article 14", "is_borderline": false}
{"id": "bench_002", "sentence": "Candidate rejections are executed autonomously with no human oversight mechanism.", "gold_category": "HUMAN_OVERSIGHT", "gold_status": "ABSENT", "target_article": "Article 14", "is_borderline": false}
{"id": "bench_003", "sentence": "A manual override capability is currently in development and planned for future release in Q4.", "gold_category": "HUMAN_OVERSIGHT", "gold_status": "PLANNED", "target_article": "Article 14", "is_borderline": false}
{"id": "bench_004", "sentence": "Bias examination and demographic parity across patient demographics was verified with zero disparity.", "gold_category": "BIAS_MITIGATION", "gold_status": "IMPLEMENTED", "target_article": "Article 10(2)(f)", "is_borderline": false}
{"id": "bench_005", "sentence": "The development team did not conduct bias examination or demographic parity audits.", "gold_category": "BIAS_MITIGATION", "gold_status": "ABSENT", "target_article": "Article 10(2)(f)", "is_borderline": false}
{"id": "bench_006", "sentence": "Disparate impact testing across immigrant zip codes is scheduled for evaluation next quarter.", "gold_category": "BIAS_MITIGATION", "gold_status": "PLANNED", "target_article": "Article 10(2)(f)", "is_borderline": false}
{"id": "bench_007", "sentence": "A continuous risk management system is implemented in accordance with ISO 14971.", "gold_category": "RISK_MANAGEMENT", "gold_status": "IMPLEMENTED", "target_article": "Article 9", "is_borderline": false}
{"id": "bench_008", "sentence": "The prototype operates without risk management procedures or hazard logs.", "gold_category": "RISK_MANAGEMENT", "gold_status": "ABSENT", "target_article": "Article 9", "is_borderline": false}
{"id": "bench_009", "sentence": "Training data provenance is documented across multi-center clinical cohorts comprising 45,000 scans.", "gold_category": "DATA_GOVERNANCE", "gold_status": "IMPLEMENTED", "target_article": "Article 10", "is_borderline": false}
{"id": "bench_010", "sentence": "Dataset lineage records were lost and no data governance protocols were applied.", "gold_category": "DATA_GOVERNANCE", "gold_status": "ABSENT", "target_article": "Article 10", "is_borderline": false}
{"id": "bench_011", "sentence": "Automated logging of every inference request and model confidence score is active.", "gold_category": "RECORD_KEEPING", "gold_status": "IMPLEMENTED", "target_article": "Article 12", "is_borderline": false}
{"id": "bench_012", "sentence": "Audit trail logging was omitted due to database storage constraints.", "gold_category": "RECORD_KEEPING", "gold_status": "ABSENT", "target_article": "Article 12", "is_borderline": false}
{"id": "bench_013", "sentence": "An emergency stop kill switch is implemented for immediate operator intervention.", "gold_category": "FAIL_SAFE", "gold_status": "IMPLEMENTED", "target_article": "Article 14(4)(e)", "is_borderline": false}
{"id": "bench_014", "sentence": "The system lacks fail-safe fallback triggers or automated circuit breakers.", "gold_category": "FAIL_SAFE", "gold_status": "ABSENT", "target_article": "Article 14(4)(e)", "is_borderline": false}
{"id": "bench_015", "sentence": "Cybersecurity controls and defenses against adversarial image perturbations are deployed.", "gold_category": "CYBERSECURITY", "gold_status": "IMPLEMENTED", "target_article": "Article 15(4)", "is_borderline": false}
{"id": "bench_016", "sentence": "Adversarial testing against resume prompt injection was untested.", "gold_category": "CYBERSECURITY", "gold_status": "ABSENT", "target_article": "Article 15(4)", "is_borderline": false}
{"id": "bench_017", "sentence": "Robustness testing against feature noise manipulation is under evaluation.", "gold_category": "ACCURACY_ROBUSTNESS", "gold_status": "PLANNED", "target_article": "Article 15(1)", "is_borderline": false}
{"id": "bench_018", "sentence": "Comprehensive instructions for use and model card transparency disclosures are provided.", "gold_category": "TRANSPARENCY", "gold_status": "IMPLEMENTED", "target_article": "Article 13", "is_borderline": false}
{"id": "bench_019", "sentence": "Technical documentation conforming to Annex IV is maintained and archived.", "gold_category": "TECHNICAL_DOCUMENTATION", "gold_status": "IMPLEMENTED", "target_article": "Article 11", "is_borderline": false}
{"id": "bench_020", "sentence": "Human supervisor review might be considered in future releases if budget permits.", "gold_category": "HUMAN_OVERSIGHT", "gold_status": "PLANNED", "target_article": "Article 14", "is_borderline": true}