| | from mmengine.config import read_base |
| |
|
| | from opencompass.models import (HuggingFacewithChatTemplate, |
| | TurboMindModelwithChatTemplate) |
| | from opencompass.utils.text_postprocessors import extract_non_reasoning_content |
| |
|
| | with read_base(): |
| | |
| | from opencompass.configs.chatml_datasets.C_MHChem.C_MHChem_gen import \ |
| | datasets as C_MHChem_chatml_datasets |
| | from opencompass.configs.chatml_datasets.CPsyExam.CPsyExam_gen import \ |
| | datasets as CPsyExam_chatml_datasets |
| | from opencompass.configs.chatml_datasets.MaScQA.MaScQA_gen import \ |
| | datasets as MaScQA_chatml_datasets |
| | from opencompass.configs.chatml_datasets.UGPhysics.UGPhysics_gen import \ |
| | datasets as UGPhysics_chatml_datasets |
| | from opencompass.configs.datasets.eese.eese_llm_judge_gen import \ |
| | eese_datasets |
| |
|
| | from ...rjob import eval, infer |
| |
|
| | chatml_datasets = [ |
| | v[0] for k, v in locals().items() |
| | if k.endswith('_chatml_datasets') and isinstance(v, list) and len(v) > 0 |
| | ] |
| |
|
| | datasets = [eese_datasets[0]] |
| |
|
| | for d in chatml_datasets: |
| | d['test_range'] = '[0:16]' |
| |
|
| | for d in datasets: |
| | if 'reader_cfg' in d: |
| | d['reader_cfg']['test_range'] = '[0:16]' |
| | else: |
| | d['test_range'] = '[0:16]' |
| | if 'eval_cfg' in d and 'dataset_cfg' in d['eval_cfg'][ |
| | 'evaluator'] and 'reader_cfg' in d['eval_cfg']['evaluator'][ |
| | 'dataset_cfg']: |
| | d['eval_cfg']['evaluator']['dataset_cfg']['reader_cfg'][ |
| | 'test_range'] = '[0:16]' |
| | if 'eval_cfg' in d and 'llm_evaluator' in d['eval_cfg'][ |
| | 'evaluator'] and 'dataset_cfg' in d['eval_cfg']['evaluator'][ |
| | 'llm_evaluator']: |
| | d['eval_cfg']['evaluator']['llm_evaluator']['dataset_cfg'][ |
| | 'reader_cfg']['test_range'] = '[0:16]' |
| |
|
| | hf_model = dict(type=HuggingFacewithChatTemplate, |
| | abbr='qwen-3-8b-hf-fullbench', |
| | path='Qwen/Qwen3-8B', |
| | max_out_len=8192, |
| | batch_size=8, |
| | run_cfg=dict(num_gpus=1), |
| | pred_postprocessor=dict(type=extract_non_reasoning_content)) |
| |
|
| | tm_model = dict(type=TurboMindModelwithChatTemplate, |
| | abbr='qwen-3-8b-fullbench', |
| | path='Qwen/Qwen3-8B', |
| | engine_config=dict(session_len=32768, max_batch_size=1, tp=1), |
| | gen_config=dict(do_sample=False, enable_thinking=True), |
| | max_seq_len=32768, |
| | max_out_len=32768, |
| | batch_size=1, |
| | run_cfg=dict(num_gpus=1), |
| | pred_postprocessor=dict(type=extract_non_reasoning_content)) |
| |
|
| | models = [hf_model, tm_model] |
| |
|
| | models = sorted(models, key=lambda x: x['run_cfg']['num_gpus']) |
| |
|
| | obj_judge_model = dict( |
| | type=TurboMindModelwithChatTemplate, |
| | abbr='qwen-3-8b-fullbench', |
| | path='Qwen/Qwen3-8B', |
| | engine_config=dict(session_len=46000, max_batch_size=1, tp=1), |
| | gen_config=dict(do_sample=False, enable_thinking=True), |
| | max_seq_len=46000, |
| | max_out_len=46000, |
| | batch_size=1, |
| | run_cfg=dict(num_gpus=1), |
| | pred_postprocessor=dict(type=extract_non_reasoning_content)) |
| |
|
| | for d in datasets: |
| | if 'eval_cfg' in d and 'evaluator' in d['eval_cfg']: |
| | if 'judge_cfg' in d['eval_cfg']['evaluator']: |
| | d['eval_cfg']['evaluator']['judge_cfg'] = obj_judge_model |
| | if 'llm_evaluator' in d['eval_cfg']['evaluator'] and 'judge_cfg' in d[ |
| | 'eval_cfg']['evaluator']['llm_evaluator']: |
| | d['eval_cfg']['evaluator']['llm_evaluator'][ |
| | 'judge_cfg'] = obj_judge_model |
| |
|
| | for d in chatml_datasets: |
| | if 'judge_cfg' in d['evaluator']: |
| | d['evaluator']['judge_cfg'] = obj_judge_model |
| | if 'llm_evaluator' in d['evaluator'] and 'judge_cfg' in d['evaluator'][ |
| | 'llm_evaluator']: |
| | d['evaluator']['llm_evaluator']['judge_cfg'] = obj_judge_model |
| |
|
