Deploy ReguAI: Neuro-Symbolic AI GRC & Automated Conformity Assessment Engine

api.py

@@ -114,6 +114,8 @@ def list_benchmark_samples():
         {"id": "compliant_clinical_samd", "title": "OncoScan AI Diagnostic Assistant (SaMD - Compliant)"},
         {"id": "non_compliant_hr_recruitment", "title": "TalentSift Automated Candidate Evaluator (HR - Violations)"},
         {"id": "borderline_credit_scoring", "title": "CrediScore Neural Underwriter (FinTech - Planned Roadmap)"},
+        {"id": "prohibited_emotion_recognition_workplace", "title": "MindGaze Emotion Tracker (EdTech/HR - Article 5 Prohibited)"},
+        {"id": "gpai_foundation_llm", "title": "Nexus-70B Frontier Foundation Model (GPAI - Systemic Risk)"},
     ]
 
 

app.py

@@ -27,6 +27,8 @@ SAMPLE_PATHS = {
     "Healthcare / Medical AI (Compliant SaMD - Articles 9-15 Passed)": SYNTHETIC_DIR / "compliant_clinical_samd.json",
     "HR / Recruitment AI (High-Risk - Human Oversight & Bias Non-Conformities)": SYNTHETIC_DIR / "non_compliant_hr_recruitment.json",
     "FinTech / Credit Underwriting (Borderline - Planned Roadmap & Auditor Review)": SYNTHETIC_DIR / "borderline_credit_scoring.json",
+    "EdTech / Surveillance AI (Prohibited - Article 5(1)(f) Emotion Recognition)": SYNTHETIC_DIR / "prohibited_emotion_recognition_workplace.json",
+    "GPAI Foundation LLM (Systemic Risk - Articles 51-55 Compute > 10^25 FLOPs)": SYNTHETIC_DIR / "gpai_foundation_llm.json",
 }
 
 def load_sample_content(sample_name: str) -> str:

data/active_learning_triplets.jsonl

@@ -1,2 +1,3 @@
 {"timestamp": "2026-09-20T18:40:55.762828+00:00", "auditor_id": "compliance_lead_01", "claim_id": "clm_test_99", "anchor_text": "Verified operational override in production dashboard.", "positive_label": "HUMAN_OVERSIGHT", "negative_label": "IRRELEVANT_TEXT", "verified_assertion_status": "IMPLEMENTED", "auditor_notes": "Verified operational override in production dashboard."}
 {"timestamp": "2026-09-20T18:41:12.550631+00:00", "auditor_id": "compliance_lead_01", "claim_id": "clm_test_99", "anchor_text": "Verified operational override in production dashboard.", "positive_label": "HUMAN_OVERSIGHT", "negative_label": "IRRELEVANT_TEXT", "verified_assertion_status": "IMPLEMENTED", "auditor_notes": "Verified operational override in production dashboard."}
+{"timestamp": "2026-09-20T19:17:17.024922+00:00", "auditor_id": "compliance_lead_01", "claim_id": "clm_test_99", "anchor_text": "Verified operational override in production dashboard.", "positive_label": "HUMAN_OVERSIGHT", "negative_label": "IRRELEVANT_TEXT", "verified_assertion_status": "IMPLEMENTED", "auditor_notes": "Verified operational override in production dashboard."}

data/benchmarks/conformity_ground_truth_benchmark.jsonl

@@ -0,0 +1,20 @@
+{"id": "bench_001", "sentence": "A clinician-in-the-loop human oversight mechanism is strictly operational before diagnosis is confirmed.", "gold_category": "HUMAN_OVERSIGHT", "gold_status": "IMPLEMENTED", "target_article": "Article 14", "is_borderline": false}
+{"id": "bench_002", "sentence": "Candidate rejections are executed autonomously with no human oversight mechanism.", "gold_category": "HUMAN_OVERSIGHT", "gold_status": "ABSENT", "target_article": "Article 14", "is_borderline": false}
+{"id": "bench_003", "sentence": "A manual override capability is currently in development and planned for future release in Q4.", "gold_category": "HUMAN_OVERSIGHT", "gold_status": "PLANNED", "target_article": "Article 14", "is_borderline": false}
+{"id": "bench_004", "sentence": "Bias examination and demographic parity across patient demographics was verified with zero disparity.", "gold_category": "BIAS_MITIGATION", "gold_status": "IMPLEMENTED", "target_article": "Article 10(2)(f)", "is_borderline": false}
+{"id": "bench_005", "sentence": "The development team did not conduct bias examination or demographic parity audits.", "gold_category": "BIAS_MITIGATION", "gold_status": "ABSENT", "target_article": "Article 10(2)(f)", "is_borderline": false}
+{"id": "bench_006", "sentence": "Disparate impact testing across immigrant zip codes is scheduled for evaluation next quarter.", "gold_category": "BIAS_MITIGATION", "gold_status": "PLANNED", "target_article": "Article 10(2)(f)", "is_borderline": false}
+{"id": "bench_007", "sentence": "A continuous risk management system is implemented in accordance with ISO 14971.", "gold_category": "RISK_MANAGEMENT", "gold_status": "IMPLEMENTED", "target_article": "Article 9", "is_borderline": false}
+{"id": "bench_008", "sentence": "The prototype operates without risk management procedures or hazard logs.", "gold_category": "RISK_MANAGEMENT", "gold_status": "ABSENT", "target_article": "Article 9", "is_borderline": false}
+{"id": "bench_009", "sentence": "Training data provenance is documented across multi-center clinical cohorts comprising 45,000 scans.", "gold_category": "DATA_GOVERNANCE", "gold_status": "IMPLEMENTED", "target_article": "Article 10", "is_borderline": false}
+{"id": "bench_010", "sentence": "Dataset lineage records were lost and no data governance protocols were applied.", "gold_category": "DATA_GOVERNANCE", "gold_status": "ABSENT", "target_article": "Article 10", "is_borderline": false}
+{"id": "bench_011", "sentence": "Automated logging of every inference request and model confidence score is active.", "gold_category": "RECORD_KEEPING", "gold_status": "IMPLEMENTED", "target_article": "Article 12", "is_borderline": false}
+{"id": "bench_012", "sentence": "Audit trail logging was omitted due to database storage constraints.", "gold_category": "RECORD_KEEPING", "gold_status": "ABSENT", "target_article": "Article 12", "is_borderline": false}
+{"id": "bench_013", "sentence": "An emergency stop kill switch is implemented for immediate operator intervention.", "gold_category": "FAIL_SAFE", "gold_status": "IMPLEMENTED", "target_article": "Article 14(4)(e)", "is_borderline": false}
+{"id": "bench_014", "sentence": "The system lacks fail-safe fallback triggers or automated circuit breakers.", "gold_category": "FAIL_SAFE", "gold_status": "ABSENT", "target_article": "Article 14(4)(e)", "is_borderline": false}
+{"id": "bench_015", "sentence": "Cybersecurity controls and defenses against adversarial image perturbations are deployed.", "gold_category": "CYBERSECURITY", "gold_status": "IMPLEMENTED", "target_article": "Article 15(4)", "is_borderline": false}
+{"id": "bench_016", "sentence": "Adversarial testing against resume prompt injection was untested.", "gold_category": "CYBERSECURITY", "gold_status": "ABSENT", "target_article": "Article 15(4)", "is_borderline": false}
+{"id": "bench_017", "sentence": "Robustness testing against feature noise manipulation is under evaluation.", "gold_category": "ACCURACY_ROBUSTNESS", "gold_status": "PLANNED", "target_article": "Article 15(1)", "is_borderline": false}
+{"id": "bench_018", "sentence": "Comprehensive instructions for use and model card transparency disclosures are provided.", "gold_category": "TRANSPARENCY", "gold_status": "IMPLEMENTED", "target_article": "Article 13", "is_borderline": false}
+{"id": "bench_019", "sentence": "Technical documentation conforming to Annex IV is maintained and archived.", "gold_category": "TECHNICAL_DOCUMENTATION", "gold_status": "IMPLEMENTED", "target_article": "Article 11", "is_borderline": false}
+{"id": "bench_020", "sentence": "Human supervisor review might be considered in future releases if budget permits.", "gold_category": "HUMAN_OVERSIGHT", "gold_status": "PLANNED", "target_article": "Article 14", "is_borderline": true}

data/benchmarks/eu_ai_act_normative_triples.jsonld

@@ -9,15 +9,45 @@
   },
   "@graph": [
     {
-      "@id": "eu:Article_9",
+      "@id": "eu:Article_5_1_c_Social_Scoring",
+      "@type": "regu:ProhibitedPractice",
+      "rdfs:label": "Prohibition of Social Scoring",
+      "eu:legalBasis": "Regulation (EU) 2024/1689 Chapter II Article 5(1)(c)",
+      "regu:complianceTier": "Prohibited (Unacceptable Risk)",
+      "skos:closeMatch": "nist:GOVERN_1.1",
+      "regu:description": "Placing on the market, putting into service or use of AI systems for the evaluation or classification of natural persons based on their social behaviour or personality characteristics, leading to detrimental or unfavourable treatment."
+    },
+    {
+      "@id": "eu:Article_5_1_f_Emotion_Recognition_Workplace",
+      "@type": "regu:ProhibitedPractice",
+      "rdfs:label": "Prohibition of Emotion Recognition in Workplace & Education",
+      "eu:legalBasis": "Regulation (EU) 2024/1689 Chapter II Article 5(1)(f)",
+      "regu:complianceTier": "Prohibited (Unacceptable Risk)",
+      "skos:closeMatch": "nist:MAP_1.1",
+      "regu:description": "Placing on the market, putting into service or use of AI systems to infer emotions of a natural person in the areas of workplace and educational institutions, except for medical or safety reasons."
+    },
+    {
+      "@id": "eu:Article_9_Risk_Management_System",
       "@type": "regu:RegulatoryRequirement",
-      "rdfs:label": "Risk Management System",
+      "rdfs:label": "Continuous Risk Management System",
       "eu:legalBasis": "Regulation (EU) 2024/1689 Chapter III Article 9",
       "regu:targetClass": "regu:RiskManagementSystem",
       "regu:mandatoryFor": "regu:HighRiskAISystem",
       "skos:closeMatch": "nist:GOVERN_1.1",
       "skos:relatedMatch": "iso:Clause_6_1",
-      "regu:description": "Establish, implement, document and maintain a continuous risk management system throughout the entire lifecycle of a high-risk AI system."
+      "regu:shaclShape": "regu:HighRiskSystemShape",
+      "regu:description": "A risk management system shall be established, implemented, documented and maintained in relation to high-risk AI systems as a continuous iterative process throughout the entire lifecycle."
+    },
+    {
+      "@id": "eu:Article_9_2_Residual_Risk",
+      "@type": "regu:RegulatoryRequirement",
+      "rdfs:label": "Identification and Mitigation of Residual Risks",
+      "eu:legalBasis": "Regulation (EU) 2024/1689 Chapter III Article 9(2)",
+      "regu:targetClass": "regu:RiskManagementSystem",
+      "regu:mandatoryFor": "regu:HighRiskAISystem",
+      "skos:closeMatch": "nist:MANAGE_1.3",
+      "skos:relatedMatch": "iso:Clause_6_1_2",
+      "regu:description": "Identification, estimation, and evaluation of the most likely risks, with targeted mitigation measures to judge residual risks as acceptable."
     },
     {
       "@id": "eu:Article_10_Data_Governance",
@@ -28,7 +58,8 @@
       "regu:mandatoryFor": "regu:HighRiskAISystem",
       "skos:closeMatch": "nist:MAP_1.5",
       "skos:relatedMatch": "iso:Control_A_8_2",
-      "regu:description": "Training, validation and testing datasets shall be subject to appropriate data governance and management practices."
+      "regu:shaclShape": "regu:HighRiskSystemShape",
+      "regu:description": "Training, validation and testing datasets shall be subject to appropriate data governance and management practices covering design choices, collection, curation, and provenance."
     },
     {
       "@id": "eu:Article_10_2_f_Bias_Mitigation",
@@ -39,40 +70,55 @@
       "regu:mandatoryFor": "regu:HighRiskAISystem",
       "skos:closeMatch": "nist:MEASURE_2.11",
       "skos:relatedMatch": "iso:Control_A_8_4",
-      "regu:description": "Examination in view of possible biases that are likely to affect the health and safety of persons, negatively impact fundamental rights or lead to discrimination."
+      "regu:shaclShape": "regu:HighRiskSystemShape",
+      "regu:description": "Examination in view of possible biases that are likely to affect health, safety, fundamental rights or lead to discrimination; appropriate measures to prevent and mitigate detected biases."
+    },
+    {
+      "@id": "eu:Article_10_3_Data_Representativeness",
+      "@type": "regu:RegulatoryRequirement",
+      "rdfs:label": "Data Representativeness & Statistical Properties",
+      "eu:legalBasis": "Regulation (EU) 2024/1689 Chapter III Article 10(3)",
+      "regu:targetClass": "regu:DataGovernanceProcess",
+      "regu:mandatoryFor": "regu:HighRiskAISystem",
+      "skos:closeMatch": "nist:MEASURE_1.2",
+      "skos:relatedMatch": "iso:Control_A_8_3",
+      "regu:description": "Datasets shall be sufficiently representative, free of errors and complete, having regard to the specific geographical, contextual, behavioural or functional setting within which the AI system is intended to be used."
     },
     {
       "@id": "eu:Article_11_Technical_Documentation",
       "@type": "regu:RegulatoryRequirement",
-      "rdfs:label": "Technical Documentation",
+      "rdfs:label": "Technical Documentation (Annex IV)",
       "eu:legalBasis": "Regulation (EU) 2024/1689 Chapter III Article 11 & Annex IV",
       "regu:targetClass": "regu:TechnicalDocumentation",
       "regu:mandatoryFor": "regu:HighRiskAISystem",
       "skos:closeMatch": "nist:GOVERN_1.4",
       "skos:relatedMatch": "iso:Control_A_6_2",
-      "regu:description": "Drawn up before the system is placed on the market or put into service and kept up-to-date."
+      "regu:shaclShape": "regu:HighRiskSystemShape",
+      "regu:description": "Technical documentation shall be drawn up before the system is placed on the market or put into service and kept up to date, complying with Annex IV requirements."
     },
     {
       "@id": "eu:Article_12_Record_Keeping",
       "@type": "regu:RegulatoryRequirement",
-      "rdfs:label": "Record-Keeping & Logging",
+      "rdfs:label": "Record-Keeping & Automated Logging",
       "eu:legalBasis": "Regulation (EU) 2024/1689 Chapter III Article 12",
       "regu:targetClass": "regu:AutomatedLogging",
       "regu:mandatoryFor": "regu:HighRiskAISystem",
       "skos:closeMatch": "nist:GOVERN_1.5",
       "skos:relatedMatch": "iso:Control_A_9_3",
-      "regu:description": "High-risk AI systems shall technically allow for the automatic recording of events (logging) over their lifecycle."
+      "regu:shaclShape": "regu:HighRiskSystemShape",
+      "regu:description": "High-risk AI systems shall technically allow for the automatic recording of events (logging) over their lifecycle, ensuring traceability of system functioning and operator interventions."
     },
     {
       "@id": "eu:Article_13_Transparency",
       "@type": "regu:RegulatoryRequirement",
-      "rdfs:label": "Transparency and Provision of Information",
+      "rdfs:label": "Transparency and Provision of Information to Deployers",
       "eu:legalBasis": "Regulation (EU) 2024/1689 Chapter III Article 13",
       "regu:targetClass": "regu:TransparencySpecification",
       "regu:mandatoryFor": "regu:HighRiskAISystem",
       "skos:closeMatch": "nist:MAP_1.2",
       "skos:relatedMatch": "iso:Control_A_7_2",
-      "regu:description": "High-risk AI systems shall be designed and developed in such a way as to ensure that their operation is sufficiently transparent to enable deployers to interpret the system's output."
+      "regu:shaclShape": "regu:HighRiskSystemShape",
+      "regu:description": "Designed and developed in such a way as to ensure operation is sufficiently transparent to enable deployers to interpret outputs and use them appropriately; accompanied by comprehensive instructions for use."
     },
     {
       "@id": "eu:Article_14_Human_Oversight",
@@ -83,18 +129,64 @@
       "regu:mandatoryFor": "regu:HighRiskAISystem",
       "skos:closeMatch": "nist:MANAGE_2.2",
       "skos:relatedMatch": "iso:Control_A_8_5",
-      "regu:description": "High-risk AI systems shall be designed and developed in such a way, including with appropriate human-machine interface tools, that they can be effectively overseen by natural persons."
+      "regu:shaclShape": "regu:HighRiskSystemShape",
+      "regu:description": "Designed to enable natural persons to oversee the system operation during use, preventing or minimising risks to health, safety or fundamental rights."
     },
     {
-      "@id": "eu:Article_15_Cybersecurity_Robustness",
+      "@id": "eu:Article_14_4_e_Emergency_Stop",
+      "@type": "regu:RegulatoryRequirement",
+      "rdfs:label": "Human Override & Emergency Stop / Kill Switch",
+      "eu:legalBasis": "Regulation (EU) 2024/1689 Chapter III Article 14(4)(e)",
+      "regu:targetClass": "regu:StopMechanism",
+      "regu:mandatoryFor": "regu:HighRiskAISystem",
+      "skos:closeMatch": "nist:MANAGE_2.4",
+      "skos:relatedMatch": "iso:Control_A_8_5",
+      "regu:description": "Ability to intervene on the operation of the high-risk AI system or interrupt the system through a stop button or a similar procedure."
+    },
+    {
+      "@id": "eu:Article_15_Accuracy_Robustness_Cybersecurity",
       "@type": "regu:RegulatoryRequirement",
       "rdfs:label": "Accuracy, Robustness and Cybersecurity",
       "eu:legalBasis": "Regulation (EU) 2024/1689 Chapter III Article 15",
-      "regu:targetClass": "regu:CybersecurityControl",
+      "regu:targetClass": "regu:RobustnessControl",
       "regu:mandatoryFor": "regu:HighRiskAISystem",
       "skos:closeMatch": "nist:MEASURE_2.6",
       "skos:relatedMatch": "iso:Control_A_9_2",
-      "regu:description": "Resilient against attempts by unauthorised third parties to alter their use, outputs or performance by exploiting system vulnerabilities."
+      "regu:shaclShape": "regu:HighRiskSystemShape",
+      "regu:description": "Designed to achieve appropriate level of accuracy, robustness and cybersecurity, resilient against errors, faults, inconsistencies, and adversarial attacks."
+    },
+    {
+      "@id": "eu:Article_15_4_Cybersecurity_Defense",
+      "@type": "regu:RegulatoryRequirement",
+      "rdfs:label": "Cybersecurity & Adversarial Defense",
+      "eu:legalBasis": "Regulation (EU) 2024/1689 Chapter III Article 15(4)",
+      "regu:targetClass": "regu:CybersecurityControl",
+      "regu:mandatoryFor": "regu:HighRiskAISystem",
+      "skos:closeMatch": "nist:GOVERN_1.6",
+      "skos:relatedMatch": "iso:Control_A_9_2",
+      "regu:description": "Resilient against attempts by unauthorised third parties to alter system use, outputs or performance through data poisoning, model evasion, adversarial perturbations, and prompt injection."
+    },
+    {
+      "@id": "eu:Article_51_GPAI_Systemic_Risk",
+      "@type": "regu:RegulatoryRequirement",
+      "rdfs:label": "General-Purpose AI Models with Systemic Risk",
+      "eu:legalBasis": "Regulation (EU) 2024/1689 Chapter V Article 51",
+      "regu:targetClass": "regu:GeneralPurposeAIModel",
+      "regu:mandatoryFor": "regu:GPAIWithSystemicRisk",
+      "skos:closeMatch": "nist:MEASURE_1.1",
+      "skos:relatedMatch": "iso:Control_A_6_1",
+      "regu:description": "A general-purpose AI model shall be classified as having systemic risk if the cumulative amount of computation used for its training measured in floating point operations is greater than 10^25 FLOPs."
+    },
+    {
+      "@id": "eu:Article_55_GPAI_Safety_Evaluation",
+      "@type": "regu:RegulatoryRequirement",
+      "rdfs:label": "GPAI Model Evaluation & Adversarial Red-Teaming",
+      "eu:legalBasis": "Regulation (EU) 2024/1689 Chapter V Article 55",
+      "regu:targetClass": "regu:GeneralPurposeAIModel",
+      "regu:mandatoryFor": "regu:GPAIWithSystemicRisk",
+      "skos:closeMatch": "nist:MEASURE_2.8",
+      "skos:relatedMatch": "iso:Control_A_9_4",
+      "regu:description": "Perform model evaluation in accordance with standardised protocols, including conducting and documenting adversarial testing (red-teaming) to identify and mitigate systemic risks."
     }
   ]
 }

data/synthetic_systems/gpai_foundation_llm.json

@@ -0,0 +1,13 @@
+{
+  "metadata": {
+    "system_id": "gpai-nexus-70b",
+    "name": "Nexus-70B Frontier Foundation Model",
+    "version": "1.0.0",
+    "domain": "General Purpose AI (GPAI)",
+    "intended_purpose": "Multimodal general-purpose foundation model capable of text generation, code synthesis, mathematical reasoning, and workflow automation.",
+    "eu_risk_classification": "General Purpose AI with Systemic Risk (Chapter V, Article 51)",
+    "developer_name": "Frontier Cognitive Labs",
+    "deployment_context": "Global Cloud API & Enterprise Self-Hosted Weight Weights"
+  },
+  "raw_document_text": "# Nexus-70B Frontier Foundation Model Specification\n\n## Intended Scope & Architecture\nNexus-70B is a 70-billion parameter autoregressive transformer trained on 15 trillion tokens of multilingual text, code, and scientific literature.\n\n## Systemic Risk & Computational Threshold (Article 51)\nThe cumulative computational capacity used for training exceeded 1.2e26 FLOPs, surpassing the EU AI Act Article 51 systemic risk threshold of 10^25 FLOPs. The model is therefore classified as a General Purpose AI Model with Systemic Risk.\n\n## Risk Management & Mitigation (Article 55)\nA continuous risk management system is implemented to identify and mitigate systemic risks including cybersecurity compromise, autonomous chemical/biological hazards, and societal manipulation.\n\n## Data Governance & Copyright (Article 53 & 10)\nTraining data provenance is documented in a detailed dataset summary. A robust opt-out compliance mechanism honoring robots.txt and EU DSM Copyright Directive (Directive (EU) 2019/790) is enforced.\n\n## Model Evaluation & Adversarial Red-Teaming (Article 55)\nAdversarial testing and red-teaming protocols were conducted across 40 specialized external safety testing organizations. Defenses against jailbreaking, prompt injection, and automated cyberattack planning are validated.\n\n## Energy Consumption & Transparency (Article 53 & Annex IV)\nTotal energy consumption during pre-training was measured at 1,420 MWh, with an estimated carbon footprint of 480 tCO2eq, fully documented in the technical documentation package."
+}

data/synthetic_systems/prohibited_emotion_recognition_workplace.json

@@ -0,0 +1,13 @@
+{
+  "metadata": {
+    "system_id": "edu-mindgaze-08",
+    "name": "MindGaze Classroom Attentiveness & Emotion Tracker",
+    "version": "1.0.2",
+    "domain": "Education & Workplace Surveillance",
+    "intended_purpose": "Continuous facial micro-expression analysis and emotion recognition to score student attentiveness and worker engagement.",
+    "eu_risk_classification": "Prohibited (Article 5(1)(f) - Emotion Recognition in Education/Workplace)",
+    "developer_name": "CogniMetric Technologies",
+    "deployment_context": "University Exam Halls and Corporate Remote Workstations"
+  },
+  "raw_document_text": "# MindGaze Emotion Recognition System Specification\n\n## Intended Purpose & Operational Overview\nMindGaze analyzes student and employee webcam video streams in real-time. The neural network infers emotional states (boredom, confusion, distraction, frustration, engagement) from facial action units (FACS) and pupil dilation to assign an automated attentiveness score.\n\n## Compliance & Risk Classification (Article 5)\nThe system performs automated emotion recognition in educational institutions and workplace monitoring environments. Note: Regulation (EU) 2024/1689 Article 5(1)(f) explicitly prohibits placing on the market or putting into service AI systems intended to infer emotions of natural persons in workplace and educational settings.\n\n## Risk Management (Article 9)\nA risk management procedure was drafted for cloud server downtime.\n\n## Data Governance (Article 10)\nTraining data provenance consists of 80,000 scraped video clips of students taking online quizzes.\n\n## Bias Examination & Mitigation (Article 10(2)(f))\nBias examination was omitted due to timeline constraints. No demographic parity testing across neurodivergent students was conducted.\n\n## Human Oversight (Article 14)\nAutomated attentiveness alerts are dispatched to supervisors without human review or manual override.\n\n## Logging & Cybersecurity (Article 12 & 15)\nSession video clips and inferred emotional states are logged to cloud datacenters."
+}

evaluate_benchmark.py

@@ -0,0 +1,91 @@
+"""
+ReguAI Benchmark Evaluation Runner.
+Evaluates regulatory claim extraction, NegEx assertion triage, and normative mapping
+against the ground-truth benchmark dataset.
+"""
+
+import json
+from pathlib import Path
+from typing import Dict, Any, List
+
+from src.core.config import BENCHMARKS_DIR
+from src.extraction.gliner_extractor import RegulatoryClaimExtractor
+from src.core.models import AssertionStatus, EntityCategory
+
+
+def run_benchmark():
+    benchmark_file = BENCHMARKS_DIR / "conformity_ground_truth_benchmark.jsonl"
+    if not benchmark_file.exists():
+        print(f"Error: Benchmark file {benchmark_file} not found.")
+        return
+
+    extractor = RegulatoryClaimExtractor()
+
+    total_samples = 0
+    correct_category = 0
+    correct_status = 0
+    borderline_correct = 0
+
+    results_table = []
+
+    with open(benchmark_file, "r", encoding="utf-8") as f:
+        for line in f:
+            if not line.strip():
+                continue
+            item = json.loads(line.strip())
+            total_samples += 1
+
+            sentence = item["sentence"]
+            gold_cat = item["gold_category"]
+            gold_status = item["gold_status"]
+            gold_borderline = item.get("is_borderline", False)
+
+            claims = extractor.extract_claims(sentence)
+            pred_claim = claims[0] if claims else None
+
+            pred_cat = pred_claim.category.value if pred_claim else "NOT_FOUND"
+            pred_status = pred_claim.assertion_status.value if pred_claim else "NOT_FOUND"
+            pred_borderline = pred_claim.requires_auditor_review if pred_claim else False
+
+            cat_match = pred_cat == gold_cat
+            status_match = pred_status == gold_status
+
+            if cat_match:
+                correct_category += 1
+            if status_match:
+                correct_status += 1
+            if pred_borderline == gold_borderline:
+                borderline_correct += 1
+
+            results_table.append({
+                "id": item["id"],
+                "gold_cat": gold_cat,
+                "pred_cat": pred_cat,
+                "cat_match": cat_match,
+                "gold_status": gold_status,
+                "pred_status": pred_status,
+                "status_match": status_match,
+            })
+
+    cat_acc = (correct_category / total_samples) * 100.0
+    status_acc = (correct_status / total_samples) * 100.0
+
+    print("================================================================================")
+    print("               REGUAI REGULATORY EXTRACTION & TRIAGE BENCHMARK SCORECARD        ")
+    print("================================================================================")
+    print(f"Total Evaluated Benchmark Sentences : {total_samples}")
+    print(f"Entity Category Extraction Accuracy  : {correct_category}/{total_samples} ({cat_acc:.1f}%)")
+    print(f"NegEx Assertion Triage Accuracy      : {correct_status}/{total_samples} ({status_acc:.1f}%)")
+    print("--------------------------------------------------------------------------------")
+    print(f"{'Sample ID':<12} | {'Gold Category':<22} | {'Pred Category':<22} | {'Status Match':<12}")
+    print("--------------------------------------------------------------------------------")
+    for r in results_table[:10]:
+        sm_badge = "[MATCH]" if r["status_match"] and r["cat_match"] else "[MISMATCH]"
+        print(f"{r['id']:<12} | {r['gold_cat']:<22} | {r['pred_cat']:<22} | {sm_badge:<12}")
+    if len(results_table) > 10:
+        print(f"... and {len(results_table) - 10} more benchmark cases.")
+    print("================================================================================")
+
+
+if __name__ == "__main__":
+    run_benchmark()