| { | |
| "model": "facebook/bart-base", | |
| "dataset": "spider", | |
| "run": 1, | |
| "eval_beams": 4, | |
| "n_samples": 1034, | |
| "em_raw": 0.2495, | |
| "em_norm": 0.3095, | |
| "ex": 0.3404, | |
| "ex_errors": 551, | |
| "by_sql_type": { | |
| "AGGREGATION": { | |
| "total": 551, | |
| "em_norm": 0.3158, | |
| "ex": 0.343 | |
| }, | |
| "ORDER_BY": { | |
| "total": 237, | |
| "em_norm": 0.2405, | |
| "ex": 0.2911 | |
| }, | |
| "LIMIT": { | |
| "total": 189, | |
| "em_norm": 0.164, | |
| "ex": 0.2063 | |
| }, | |
| "SIMPLE": { | |
| "total": 185, | |
| "em_norm": 0.4919, | |
| "ex": 0.5027 | |
| }, | |
| "GROUP_BY": { | |
| "total": 277, | |
| "em_norm": 0.1661, | |
| "ex": 0.1949 | |
| }, | |
| "NESTED": { | |
| "total": 83, | |
| "em_norm": 0.2289, | |
| "ex": 0.2651 | |
| }, | |
| "JOIN": { | |
| "total": 408, | |
| "em_norm": 0.0049, | |
| "ex": 0.0417 | |
| }, | |
| "HAVING": { | |
| "total": 79, | |
| "em_norm": 0.1139, | |
| "ex": 0.1519 | |
| } | |
| }, | |
| "by_hardness": { | |
| "easy": { | |
| "total": 456, | |
| "em_norm": 0.557, | |
| "ex": 0.5811 | |
| }, | |
| "medium": { | |
| "total": 402, | |
| "em_norm": 0.0945, | |
| "ex": 0.1318 | |
| }, | |
| "hard": { | |
| "total": 168, | |
| "em_norm": 0.1667, | |
| "ex": 0.2024 | |
| }, | |
| "extra_hard": { | |
| "total": 8, | |
| "em_norm": 0.0, | |
| "ex": 0.0 | |
| } | |
| }, | |
| "error_types": { | |
| "WRONG_COLUMNS": 158, | |
| "OTHER": 91, | |
| "WRONG_TABLE": 291, | |
| "EXTRA_JOIN": 41, | |
| "MISSING_GROUP_BY": 18, | |
| "MISSING_WHERE": 7, | |
| "MISSING_ORDER_BY": 11, | |
| "MISSING_JOIN": 97 | |
| } | |
| } |