| { |
| "target_cwes": [ |
| "safe", |
| "CWE-20", |
| "CWE-22", |
| "CWE-78", |
| "CWE-79", |
| "CWE-89", |
| "CWE-94", |
| "CWE-119", |
| "CWE-125", |
| "CWE-190", |
| "CWE-200", |
| "CWE-264", |
| "CWE-269", |
| "CWE-276", |
| "CWE-284", |
| "CWE-287", |
| "CWE-310", |
| "CWE-327", |
| "CWE-330", |
| "CWE-352", |
| "CWE-362", |
| "CWE-399", |
| "CWE-401", |
| "CWE-416", |
| "CWE-434", |
| "CWE-476", |
| "CWE-502", |
| "CWE-601", |
| "CWE-787", |
| "CWE-798", |
| "CWE-918" |
| ], |
| "cwe_names": { |
| "safe": "Safe Code", |
| "CWE-20": "Improper Input Validation", |
| "CWE-22": "Path Traversal", |
| "CWE-78": "OS Command Injection", |
| "CWE-79": "XSS", |
| "CWE-89": "SQL Injection", |
| "CWE-94": "Code Injection", |
| "CWE-119": "Buffer Overflow", |
| "CWE-125": "Out-of-bounds Read", |
| "CWE-190": "Integer Overflow", |
| "CWE-200": "Information Exposure", |
| "CWE-264": "Permissions", |
| "CWE-269": "Privilege Management", |
| "CWE-276": "Incorrect Permissions", |
| "CWE-284": "Access Control", |
| "CWE-287": "Authentication", |
| "CWE-310": "Crypto Issues", |
| "CWE-327": "Broken Crypto", |
| "CWE-330": "Insufficient Randomness", |
| "CWE-352": "CSRF", |
| "CWE-362": "Race Condition", |
| "CWE-399": "Resource Management", |
| "CWE-401": "Memory Leak", |
| "CWE-416": "Use After Free", |
| "CWE-434": "File Upload", |
| "CWE-476": "NULL Pointer Deref", |
| "CWE-502": "Insecure Deserialization", |
| "CWE-601": "Open Redirect", |
| "CWE-787": "Out-of-bounds Write", |
| "CWE-798": "Hardcoded Credentials", |
| "CWE-918": "SSRF" |
| }, |
| "num_labels": 31, |
| "cwe_to_owasp": { |
| "CWE-22": "A01", |
| "CWE-200": "A01", |
| "CWE-264": "A01", |
| "CWE-276": "A01", |
| "CWE-284": "A01", |
| "CWE-352": "A01", |
| "CWE-601": "A01", |
| "CWE-269": "A01", |
| "CWE-310": "A02", |
| "CWE-327": "A02", |
| "CWE-330": "A02", |
| "CWE-20": "A03", |
| "CWE-78": "A03", |
| "CWE-79": "A03", |
| "CWE-89": "A03", |
| "CWE-94": "A03", |
| "CWE-119": "A03", |
| "CWE-125": "A03", |
| "CWE-190": "A03", |
| "CWE-416": "A03", |
| "CWE-476": "A03", |
| "CWE-401": "A03", |
| "CWE-787": "A03", |
| "CWE-434": "A04", |
| "CWE-362": "A04", |
| "CWE-399": "A04", |
| "CWE-287": "A07", |
| "CWE-798": "A07", |
| "CWE-502": "A08", |
| "CWE-918": "A10" |
| }, |
| "optimized_thresholds": { |
| "safe": 0.5750000000000002, |
| "CWE-20": 0.8750000000000003, |
| "CWE-22": 0.7750000000000002, |
| "CWE-78": 0.7500000000000002, |
| "CWE-79": 0.5750000000000002, |
| "CWE-89": 0.5250000000000001, |
| "CWE-94": 0.7750000000000002, |
| "CWE-119": 0.6000000000000002, |
| "CWE-125": 0.7000000000000003, |
| "CWE-190": 0.8750000000000003, |
| "CWE-200": 0.7500000000000002, |
| "CWE-264": 0.8250000000000003, |
| "CWE-269": 0.5500000000000003, |
| "CWE-276": 0.5, |
| "CWE-284": 0.6000000000000002, |
| "CWE-287": 0.4250000000000001, |
| "CWE-310": 0.8000000000000003, |
| "CWE-327": 0.5, |
| "CWE-330": 0.7250000000000003, |
| "CWE-352": 0.5, |
| "CWE-362": 0.6250000000000002, |
| "CWE-399": 0.9250000000000004, |
| "CWE-401": 0.4250000000000001, |
| "CWE-416": 0.6000000000000002, |
| "CWE-434": 0.3000000000000001, |
| "CWE-476": 0.8000000000000003, |
| "CWE-502": 0.8750000000000003, |
| "CWE-601": 0.47500000000000014, |
| "CWE-787": 0.6000000000000002, |
| "CWE-798": 0.25000000000000006, |
| "CWE-918": 0.5 |
| }, |
| "temperature": 0.6163493394851685, |
| "eval_metrics": { |
| "eval_macro_f1": 0.47583485129569136, |
| "eval_micro_f1": 0.9426379247351114, |
| "eval_weighted_f1": 0.945370829629837, |
| "eval_macro_precision": 0.4824943179864961, |
| "eval_macro_recall": 0.5278332334756789, |
| "eval_f1_safe": 0.9820306051177853, |
| "eval_f1_CWE-20": 0.5721153846153846, |
| "eval_f1_CWE-22": 0.5, |
| "eval_f1_CWE-78": 0.0, |
| "eval_f1_CWE-79": 0.5932203389830508, |
| "eval_f1_CWE-89": 0.7916666666666666, |
| "eval_f1_CWE-94": 0.7068273092369478, |
| "eval_f1_CWE-119": 0.7054610564010743, |
| "eval_f1_CWE-125": 0.5844748858447488, |
| "eval_f1_CWE-190": 0.7704918032786885, |
| "eval_f1_CWE-200": 0.7046632124352331, |
| "eval_f1_CWE-264": 0.697986577181208, |
| "eval_f1_CWE-269": 0.0, |
| "eval_f1_CWE-276": 0.0, |
| "eval_f1_CWE-284": 0.5128205128205128, |
| "eval_f1_CWE-287": 0.25, |
| "eval_f1_CWE-310": 0.5, |
| "eval_f1_CWE-327": 0.0, |
| "eval_f1_CWE-330": 0.5, |
| "eval_f1_CWE-352": 0.0, |
| "eval_f1_CWE-362": 0.7010309278350515, |
| "eval_f1_CWE-399": 0.6783625730994152, |
| "eval_f1_CWE-401": 0.5909090909090909, |
| "eval_f1_CWE-416": 0.48484848484848486, |
| "eval_f1_CWE-434": 0.0, |
| "eval_f1_CWE-476": 0.5483870967741935, |
| "eval_f1_CWE-502": 0.9, |
| "eval_f1_CWE-601": 0.8571428571428571, |
| "eval_f1_CWE-787": 0.47558386411889597, |
| "eval_f1_CWE-798": 0.14285714285714285, |
| "eval_f1_CWE-918": 0.0 |
| }, |
| "per_class_metrics": { |
| "safe": { |
| "f1": 0.9820306051177853, |
| "precision": 0.976204001767342, |
| "recall": 0.9879271798147556, |
| "threshold": 0.5750000000000002, |
| "support": 15655 |
| }, |
| "CWE-20": { |
| "f1": 0.5721153846153846, |
| "precision": 0.7, |
| "recall": 0.483739837398374, |
| "threshold": 0.8750000000000003, |
| "support": 246 |
| }, |
| "CWE-22": { |
| "f1": 0.5, |
| "precision": 1.0, |
| "recall": 0.3333333333333333, |
| "threshold": 0.7750000000000002, |
| "support": 6 |
| }, |
| "CWE-78": { |
| "f1": 0.0, |
| "precision": 0.0, |
| "recall": 0.0, |
| "threshold": 0.7500000000000002, |
| "support": 7 |
| }, |
| "CWE-79": { |
| "f1": 0.5932203389830508, |
| "precision": 0.4861111111111111, |
| "recall": 0.7608695652173914, |
| "threshold": 0.5750000000000002, |
| "support": 46 |
| }, |
| "CWE-89": { |
| "f1": 0.7916666666666666, |
| "precision": 0.6846846846846847, |
| "recall": 0.9382716049382716, |
| "threshold": 0.5250000000000001, |
| "support": 81 |
| }, |
| "CWE-94": { |
| "f1": 0.7068273092369478, |
| "precision": 0.676923076923077, |
| "recall": 0.7394957983193278, |
| "threshold": 0.7750000000000002, |
| "support": 119 |
| }, |
| "CWE-119": { |
| "f1": 0.7054610564010743, |
| "precision": 0.6127527216174183, |
| "recall": 0.8312236286919831, |
| "threshold": 0.6000000000000002, |
| "support": 474 |
| }, |
| "CWE-125": { |
| "f1": 0.5844748858447488, |
| "precision": 0.5245901639344263, |
| "recall": 0.6597938144329897, |
| "threshold": 0.7000000000000003, |
| "support": 97 |
| }, |
| "CWE-190": { |
| "f1": 0.7704918032786885, |
| "precision": 0.9038461538461539, |
| "recall": 0.6714285714285714, |
| "threshold": 0.8750000000000003, |
| "support": 70 |
| }, |
| "CWE-200": { |
| "f1": 0.7046632124352331, |
| "precision": 0.6938775510204082, |
| "recall": 0.7157894736842105, |
| "threshold": 0.7500000000000002, |
| "support": 95 |
| }, |
| "CWE-264": { |
| "f1": 0.697986577181208, |
| "precision": 0.7027027027027027, |
| "recall": 0.6933333333333334, |
| "threshold": 0.8250000000000003, |
| "support": 75 |
| }, |
| "CWE-269": { |
| "f1": 0.0, |
| "precision": 0.0, |
| "recall": 0.0, |
| "threshold": 0.5500000000000003, |
| "support": 2 |
| }, |
| "CWE-276": { |
| "f1": 0.0, |
| "precision": 0.0, |
| "recall": 0.0, |
| "threshold": 0.5, |
| "support": 0 |
| }, |
| "CWE-284": { |
| "f1": 0.5128205128205128, |
| "precision": 0.43478260869565216, |
| "recall": 0.625, |
| "threshold": 0.6000000000000002, |
| "support": 16 |
| }, |
| "CWE-287": { |
| "f1": 0.25, |
| "precision": 0.25, |
| "recall": 0.25, |
| "threshold": 0.4250000000000001, |
| "support": 4 |
| }, |
| "CWE-310": { |
| "f1": 0.5, |
| "precision": 0.7142857142857143, |
| "recall": 0.38461538461538464, |
| "threshold": 0.8000000000000003, |
| "support": 13 |
| }, |
| "CWE-327": { |
| "f1": 0.0, |
| "precision": 0.0, |
| "recall": 0.0, |
| "threshold": 0.5, |
| "support": 2 |
| }, |
| "CWE-330": { |
| "f1": 0.5, |
| "precision": 0.6, |
| "recall": 0.42857142857142855, |
| "threshold": 0.7250000000000003, |
| "support": 7 |
| }, |
| "CWE-352": { |
| "f1": 0.0, |
| "precision": 0.0, |
| "recall": 0.0, |
| "threshold": 0.5, |
| "support": 1 |
| }, |
| "CWE-362": { |
| "f1": 0.7010309278350515, |
| "precision": 0.6538461538461539, |
| "recall": 0.7555555555555555, |
| "threshold": 0.6250000000000002, |
| "support": 45 |
| }, |
| "CWE-399": { |
| "f1": 0.6783625730994152, |
| "precision": 0.7837837837837838, |
| "recall": 0.5979381443298969, |
| "threshold": 0.9250000000000004, |
| "support": 97 |
| }, |
| "CWE-401": { |
| "f1": 0.5909090909090909, |
| "precision": 0.4482758620689655, |
| "recall": 0.8666666666666667, |
| "threshold": 0.4250000000000001, |
| "support": 15 |
| }, |
| "CWE-416": { |
| "f1": 0.48484848484848486, |
| "precision": 0.4266666666666667, |
| "recall": 0.5614035087719298, |
| "threshold": 0.6000000000000002, |
| "support": 57 |
| }, |
| "CWE-434": { |
| "f1": 0.0, |
| "precision": 0.0, |
| "recall": 0.0, |
| "threshold": 0.3000000000000001, |
| "support": 0 |
| }, |
| "CWE-476": { |
| "f1": 0.5483870967741935, |
| "precision": 0.5862068965517241, |
| "recall": 0.5151515151515151, |
| "threshold": 0.8000000000000003, |
| "support": 99 |
| }, |
| "CWE-502": { |
| "f1": 0.9, |
| "precision": 0.9, |
| "recall": 0.9, |
| "threshold": 0.8750000000000003, |
| "support": 40 |
| }, |
| "CWE-601": { |
| "f1": 0.8571428571428571, |
| "precision": 0.75, |
| "recall": 1.0, |
| "threshold": 0.47500000000000014, |
| "support": 3 |
| }, |
| "CWE-787": { |
| "f1": 0.47558386411889597, |
| "precision": 0.3708609271523179, |
| "recall": 0.6627218934911243, |
| "threshold": 0.6000000000000002, |
| "support": 169 |
| }, |
| "CWE-798": { |
| "f1": 0.14285714285714285, |
| "precision": 0.07692307692307693, |
| "recall": 1.0, |
| "threshold": 0.25000000000000006, |
| "support": 1 |
| }, |
| "CWE-918": { |
| "f1": 0.0, |
| "precision": 0.0, |
| "recall": 0.0, |
| "threshold": 0.5, |
| "support": 0 |
| } |
| }, |
| "classification_report": " precision recall f1-score support\n\n safe 0.9762 0.9879 0.9820 15655\n CWE-20 0.7000 0.4837 0.5721 246\n CWE-22 1.0000 0.3333 0.5000 6\n CWE-78 0.0000 0.0000 0.0000 7\n CWE-79 0.4861 0.7609 0.5932 46\n CWE-89 0.6847 0.9383 0.7917 81\n CWE-94 0.6769 0.7395 0.7068 119\n CWE-119 0.6128 0.8312 0.7055 474\n CWE-125 0.5246 0.6598 0.5845 97\n CWE-190 0.9038 0.6714 0.7705 70\n CWE-200 0.6939 0.7158 0.7047 95\n CWE-264 0.7027 0.6933 0.6980 75\n CWE-269 0.0000 0.0000 0.0000 2\n CWE-276 0.0000 0.0000 0.0000 0\n CWE-284 0.4348 0.6250 0.5128 16\n CWE-287 0.2500 0.2500 0.2500 4\n CWE-310 0.7143 0.3846 0.5000 13\n CWE-327 0.0000 0.0000 0.0000 2\n CWE-330 0.6000 0.4286 0.5000 7\n CWE-352 0.0000 0.0000 0.0000 1\n CWE-362 0.6538 0.7556 0.7010 45\n CWE-399 0.7838 0.5979 0.6784 97\n CWE-401 0.4483 0.8667 0.5909 15\n CWE-416 0.4267 0.5614 0.4848 57\n CWE-434 0.0000 0.0000 0.0000 0\n CWE-476 0.5862 0.5152 0.5484 99\n CWE-502 0.9000 0.9000 0.9000 40\n CWE-601 0.7500 1.0000 0.8571 3\n CWE-787 0.3709 0.6627 0.4756 169\n CWE-798 0.0769 1.0000 0.1429 1\n CWE-918 0.0000 0.0000 0.0000 0\n\n micro avg 0.9297 0.9560 0.9426 17542\n macro avg 0.4825 0.5278 0.4758 17542\nweighted avg 0.9383 0.9560 0.9454 17542\n samples avg 0.9437 0.9560 0.9477 17542\n", |
| "improvements": [ |
| "GraphCodeBERT-base (125M, 12 layers) vs CodeBERTa-small (83M, 6 layers)", |
| "Asymmetric Loss (ASL) gamma_neg=4, gamma_pos=0, clip=0.05", |
| "Two-phase training: 4 epochs frozen + up to 9 epochs full fine-tune", |
| "Per-class threshold optimization on validation set", |
| "Temperature scaling calibration (T=0.6163)", |
| "Classification head bias initialization for imbalanced classes" |
| ], |
| "baseline_comparison": { |
| "baseline_macro_f1": 0.1157, |
| "new_macro_f1": 0.47583485129569136, |
| "improvement_pct": 311.2660771786442 |
| } |
| } |
