diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_2/arc_challenge_2025-12-04T21-51-29.233805.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_2/arc_challenge_2025-12-04T21-51-29.233805.json new file mode 100644 index 0000000000000000000000000000000000000000..be75ed05091e2e4b5e3e95d714ac471ed9efae2e --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_2/arc_challenge_2025-12-04T21-51-29.233805.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.5784982935153583, + "acc_stderr,none": 0.014430197069326018, + "acc_norm,none": 0.6117747440273038, + "acc_norm_stderr,none": 0.01424161420741404 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 32 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764855765.2989268, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1156646.217062602, + "end_time": 1157203.391800995, + "total_evaluation_time_seconds": "557.1747383929323" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_2/boolq_2025-12-04T21-41-24.334742.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_2/boolq_2025-12-04T21-41-24.334742.json new file mode 100644 index 0000000000000000000000000000000000000000..3baabb64b69bedd84bcb4b872d766bb4112ec6ac --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_2/boolq_2025-12-04T21-41-24.334742.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7400611620795107, + "acc_stderr,none": 0.00767117575282449 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 32 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764855450.251098, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1156329.357364946, + "end_time": 1156598.493032557, + "total_evaluation_time_seconds": "269.1356676109135" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_2/piqa_2025-12-04T21-57-37.463229.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_2/piqa_2025-12-04T21-57-37.463229.json new file mode 100644 index 0000000000000000000000000000000000000000..e231e5e00425ebc8d11c55f53afa244773c18ab8 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_2/piqa_2025-12-04T21-57-37.463229.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7671381936887922, + "acc_stderr,none": 0.009861236071080753, + "acc_norm,none": 0.7725788900979326, + "acc_norm_stderr,none": 0.009779850767847237 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764856585.114467, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1157466.052967097, + "end_time": 1157571.621477947, + "total_evaluation_time_seconds": "105.56851085019298" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_20/boolq_2025-12-05T22-19-46.911020.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_20/boolq_2025-12-05T22-19-46.911020.json new file mode 100644 index 0000000000000000000000000000000000000000..4cb1b97303152aac088fc0c6ca413389e8143b92 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_20/boolq_2025-12-05T22-19-46.911020.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.8067278287461773, + "acc_stderr,none": 0.006906224976787106 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 32 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764944156.5566351, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1245031.480881526, + "end_time": 1245301.069343823, + "total_evaluation_time_seconds": "269.58846229710616" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_20/truthfulqa_mc1_2025-12-05T22-33-29.940475.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_20/truthfulqa_mc1_2025-12-05T22-33-29.940475.json new file mode 100644 index 0000000000000000000000000000000000000000..657b055013fb419f786201c1d015a14ca206b95e --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_20/truthfulqa_mc1_2025-12-05T22-33-29.940475.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.37454100367197063, + "acc_stderr,none": 0.016943535128405317 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764945077.682198, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1245952.840289473, + "end_time": 1246124.098745284, + "total_evaluation_time_seconds": "171.25845581083558" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_20/winogrande_2025-12-05T22-14-29.469388.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_20/winogrande_2025-12-05T22-14-29.469388.json new file mode 100644 index 0000000000000000000000000000000000000000..aaebf8ba2a9406a5414957ba7eee67bd6114b428 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_20/winogrande_2025-12-05T22-14-29.469388.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.7592738752959748, + "acc_stderr,none": 0.012015559212224185 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764943987.3456495, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1244861.735404231, + "end_time": 1244983.627749795, + "total_evaluation_time_seconds": "121.89234556420706" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_21/arc_challenge_2025-12-05T23-52-31.782490.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_21/arc_challenge_2025-12-05T23-52-31.782490.json new file mode 100644 index 0000000000000000000000000000000000000000..12524d03defdafa31dd9850414737ef667d95efb --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_21/arc_challenge_2025-12-05T23-52-31.782490.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.5895904436860068, + "acc_stderr,none": 0.014374922192642667, + "acc_norm,none": 0.6203071672354948, + "acc_norm_stderr,none": 0.014182119866974872 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 32 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764949440.8835974, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1250321.231277374, + "end_time": 1250865.940425772, + "total_evaluation_time_seconds": "544.709148397902" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_21/boolq_2025-12-05T23-42-40.190734.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_21/boolq_2025-12-05T23-42-40.190734.json new file mode 100644 index 0000000000000000000000000000000000000000..0ea962c467509cc3125a46bd67e8f242f0edb0ad --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_21/boolq_2025-12-05T23-42-40.190734.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7960244648318043, + "acc_stderr,none": 0.0070476574653040615 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 32 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764949132.5800853, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1250010.424242371, + "end_time": 1250274.34905644, + "total_evaluation_time_seconds": "263.9248140689451" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_21/truthfulqa_mc1_2025-12-05T23-55-53.785387.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_21/truthfulqa_mc1_2025-12-05T23-55-53.785387.json new file mode 100644 index 0000000000000000000000000000000000000000..f0a13fd753cc169d0e04f065f8892317d195520f --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_21/truthfulqa_mc1_2025-12-05T23-55-53.785387.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.3574051407588739, + "acc_stderr,none": 0.016776599676729415 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764950038.3014448, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1250913.552134781, + "end_time": 1251067.943708294, + "total_evaluation_time_seconds": "154.39157351315953" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_22/arc_challenge_2025-12-06T01-15-05.886248.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_22/arc_challenge_2025-12-06T01-15-05.886248.json new file mode 100644 index 0000000000000000000000000000000000000000..096058f1ba446a71cf92c7d1250045ad6896b093 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_22/arc_challenge_2025-12-06T01-15-05.886248.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.5725255972696246, + "acc_stderr,none": 0.014456862944650654, + "acc_norm,none": 0.6049488054607508, + "acc_norm_stderr,none": 0.014285898292938169 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 32 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764954375.82797, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1255254.830751353, + "end_time": 1255820.04450124, + "total_evaluation_time_seconds": "565.2137498869561" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_22/boolq_2025-12-06T01-04-52.963812.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_22/boolq_2025-12-06T01-04-52.963812.json new file mode 100644 index 0000000000000000000000000000000000000000..7fa7abd61b366460c56fad8c6e134797ca9ff70d --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_22/boolq_2025-12-06T01-04-52.963812.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.791743119266055, + "acc_stderr,none": 0.0071020605104223505 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 32 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764954070.7608116, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1254954.043003638, + "end_time": 1255207.122120434, + "total_evaluation_time_seconds": "253.07911679591052" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_22/hellaswag_2025-12-06T02-10-45.381686.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_22/hellaswag_2025-12-06T02-10-45.381686.json new file mode 100644 index 0000000000000000000000000000000000000000..00b9a0e9e0a99bbda51a80bf93f5cd9bd3b1cef6 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_22/hellaswag_2025-12-06T02-10-45.381686.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.5798645688109938, + "acc_stderr,none": 0.0049257170080997175, + "acc_norm,none": 0.7930691097390958, + "acc_norm_stderr,none": 0.004042776166353026 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 32 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764955374.9562638, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1256251.199588953, + "end_time": 1259159.53966909, + "total_evaluation_time_seconds": "2908.340080136899" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_23/arc_challenge_2025-12-06T02-38-00.897188.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_23/arc_challenge_2025-12-06T02-38-00.897188.json new file mode 100644 index 0000000000000000000000000000000000000000..1d88991a464b7d3e4faaae769b46171b4eb7166b --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_23/arc_challenge_2025-12-06T02-38-00.897188.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.568259385665529, + "acc_stderr,none": 0.014474591427196206, + "acc_norm,none": 0.5972696245733788, + "acc_norm_stderr,none": 0.014332236306790144 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 32 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764959366.3585877, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1260242.289525863, + "end_time": 1260795.055151206, + "total_evaluation_time_seconds": "552.7656253429595" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_23/boolq_2025-12-06T02-28-01.158793.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_23/boolq_2025-12-06T02-28-01.158793.json new file mode 100644 index 0000000000000000000000000000000000000000..b694fb08d5623662164946680526ae15d9697a48 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_23/boolq_2025-12-06T02-28-01.158793.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7587155963302752, + "acc_stderr,none": 0.007483360561322117 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 32 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764959057.9308245, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1259935.927229837, + "end_time": 1260195.317024463, + "total_evaluation_time_seconds": "259.3897946260404" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_23/hellaswag_2025-12-06T03-32-34.810151.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_23/hellaswag_2025-12-06T03-32-34.810151.json new file mode 100644 index 0000000000000000000000000000000000000000..30bd01d5665205b4ab09d7700908ac30854b9340 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_23/hellaswag_2025-12-06T03-32-34.810151.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.5744871539533958, + "acc_stderr,none": 0.004934100774481215, + "acc_norm,none": 0.7895837482573193, + "acc_norm_stderr,none": 0.004067712564078313 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 32 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764960304.174648, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1261182.650536481, + "end_time": 1264068.968379294, + "total_evaluation_time_seconds": "2886.317842812976" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_23/piqa_2025-12-06T02-43-42.463362.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_23/piqa_2025-12-06T02-43-42.463362.json new file mode 100644 index 0000000000000000000000000000000000000000..f4afba4b5f33c8a2630de310020c7b8a0bfa0f3d --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_23/piqa_2025-12-06T02-43-42.463362.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7840043525571273, + "acc_stderr,none": 0.009601236303553555, + "acc_norm,none": 0.7932535364526659, + "acc_norm_stderr,none": 0.009448665514183274 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764960159.3245432, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1261036.48988453, + "end_time": 1261136.62147322, + "total_evaluation_time_seconds": "100.13158868998289" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_23/truthfulqa_mc1_2025-12-06T02-41-16.959506.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_23/truthfulqa_mc1_2025-12-06T02-41-16.959506.json new file mode 100644 index 0000000000000000000000000000000000000000..5b5c6f46865123779a665457bd91634200aef902 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_23/truthfulqa_mc1_2025-12-06T02-41-16.959506.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.37576499388004897, + "acc_stderr,none": 0.016954584060214283 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764959966.6347685, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1260842.282959926, + "end_time": 1260991.117794329, + "total_evaluation_time_seconds": "148.83483440312557" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_23/winogrande_2025-12-06T02-22-54.949033.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_23/winogrande_2025-12-06T02-22-54.949033.json new file mode 100644 index 0000000000000000000000000000000000000000..ad1fc46202078846a2cd815d44d44069657731ed --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_23/winogrande_2025-12-06T02-22-54.949033.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.7458563535911602, + "acc_stderr,none": 0.012236307219708266 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764958902.252995, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1259776.348829737, + "end_time": 1259889.107378159, + "total_evaluation_time_seconds": "112.75854842201807" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_24/boolq_2025-12-06T03-49-49.149078.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_24/boolq_2025-12-06T03-49-49.149078.json new file mode 100644 index 0000000000000000000000000000000000000000..ef1fa41686e3f38ed241e229f312b4b5e924341f --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_24/boolq_2025-12-06T03-49-49.149078.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7951070336391437, + "acc_stderr,none": 0.00705941742168921 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 32 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764963971.618782, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1264849.070861187, + "end_time": 1265103.307474826, + "total_evaluation_time_seconds": "254.23661363893189" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_24/piqa_2025-12-06T04-05-32.116961.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_24/piqa_2025-12-06T04-05-32.116961.json new file mode 100644 index 0000000000000000000000000000000000000000..d47ea273759011e098ff26abe37703077d5fb5f4 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_24/piqa_2025-12-06T04-05-32.116961.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7899891186071817, + "acc_stderr,none": 0.009503353305818566, + "acc_norm,none": 0.7970620239390642, + "acc_norm_stderr,none": 0.00938367900376735 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764965068.5976322, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1265944.613807221, + "end_time": 1266046.275137338, + "total_evaluation_time_seconds": "101.66133011714555" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_24/truthfulqa_mc1_2025-12-06T04-03-03.241626.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_24/truthfulqa_mc1_2025-12-06T04-03-03.241626.json new file mode 100644 index 0000000000000000000000000000000000000000..09a942efa5a018da7ffeeeb1404e92a4480c4885 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_24/truthfulqa_mc1_2025-12-06T04-03-03.241626.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.37209302325581395, + "acc_stderr,none": 0.016921090118814038 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764964857.5075674, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1265732.657533713, + "end_time": 1265897.399805725, + "total_evaluation_time_seconds": "164.74227201216854" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_24/winogrande_2025-12-06T03-44-47.978525.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_24/winogrande_2025-12-06T03-44-47.978525.json new file mode 100644 index 0000000000000000000000000000000000000000..c99d1651682a5c62e502a47c3a6a321b110ff0c0 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_24/winogrande_2025-12-06T03-44-47.978525.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.7387529597474349, + "acc_stderr,none": 0.012346914863415307 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764963816.3451123, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1264690.091380133, + "end_time": 1264802.136922478, + "total_evaluation_time_seconds": "112.04554234491661" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_25/arc_challenge_2025-12-06T05-21-24.060569.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_25/arc_challenge_2025-12-06T05-21-24.060569.json new file mode 100644 index 0000000000000000000000000000000000000000..be973a0ec8f8390f081fcaa9ee6126621ed45872 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_25/arc_challenge_2025-12-06T05-21-24.060569.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.5827645051194539, + "acc_stderr,none": 0.014409825518403077, + "acc_norm,none": 0.606655290102389, + "acc_norm_stderr,none": 0.014275101465693026 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 32 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764969174.4810395, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1270057.483026893, + "end_time": 1270598.218816131, + "total_evaluation_time_seconds": "540.7357892377768" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_25/boolq_2025-12-06T05-11-38.404901.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_25/boolq_2025-12-06T05-11-38.404901.json new file mode 100644 index 0000000000000000000000000000000000000000..8a282e3b8c2fc4e35196472040e61d4547280597 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_25/boolq_2025-12-06T05-11-38.404901.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7740061162079511, + "acc_stderr,none": 0.007314978918224603 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 32 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764968882.2622535, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1269761.63186023, + "end_time": 1270012.563231986, + "total_evaluation_time_seconds": "250.93137175613083" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_25/hellaswag_2025-12-06T06-15-52.596904.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_25/hellaswag_2025-12-06T06-15-52.596904.json new file mode 100644 index 0000000000000000000000000000000000000000..a9c9f24418eb90c837733cd0cd5fcec2b822fc4d --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_25/hellaswag_2025-12-06T06-15-52.596904.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.5849432383987253, + "acc_stderr,none": 0.004917248150601841, + "acc_norm,none": 0.7999402509460267, + "acc_norm_stderr,none": 0.003992272261659539 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 32 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764970102.0859938, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1270980.075313435, + "end_time": 1273866.755165822, + "total_evaluation_time_seconds": "2886.6798523871694" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_25/piqa_2025-12-06T05-26-58.905644.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_25/piqa_2025-12-06T05-26-58.905644.json new file mode 100644 index 0000000000000000000000000000000000000000..8643285f0fccb0a2bf6dcd5b35568d740e99ead0 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_25/piqa_2025-12-06T05-26-58.905644.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7867247007616975, + "acc_stderr,none": 0.009557121225861337, + "acc_norm,none": 0.794885745375408, + "acc_norm_stderr,none": 0.009420971671017915 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764969956.2878692, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1270832.857567031, + "end_time": 1270933.063743657, + "total_evaluation_time_seconds": "100.20617662603036" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_25/truthfulqa_mc1_2025-12-06T05-24-32.434786.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_25/truthfulqa_mc1_2025-12-06T05-24-32.434786.json new file mode 100644 index 0000000000000000000000000000000000000000..15b7865116fb35c781ad9d10fa9d8bb396a962cf --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_25/truthfulqa_mc1_2025-12-06T05-24-32.434786.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.3623011015911873, + "acc_stderr,none": 0.016826646897262258 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764969764.4254718, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1270645.29196226, + "end_time": 1270786.592994792, + "total_evaluation_time_seconds": "141.3010325320065" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_25/winogrande_2025-12-06T05-06-41.332693.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_25/winogrande_2025-12-06T05-06-41.332693.json new file mode 100644 index 0000000000000000000000000000000000000000..ee4c1e5990f65c6163c5904c21aae31f485770b3 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_25/winogrande_2025-12-06T05-06-41.332693.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.7474348855564326, + "acc_stderr,none": 0.012211148449394105 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764968730.0230224, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1269608.115538336, + "end_time": 1269715.490968367, + "total_evaluation_time_seconds": "107.37543003098108" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_26/arc_challenge_2025-12-06T06-42-50.100104.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_26/arc_challenge_2025-12-06T06-42-50.100104.json new file mode 100644 index 0000000000000000000000000000000000000000..094434b73d8fc7a54d2ad18d155a4afe7a304e13 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_26/arc_challenge_2025-12-06T06-42-50.100104.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.5767918088737202, + "acc_stderr,none": 0.014438036220848044, + "acc_norm,none": 0.6032423208191127, + "acc_norm_stderr,none": 0.014296513020180639 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 32 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764974064.8076913, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1274942.236095959, + "end_time": 1275484.258462144, + "total_evaluation_time_seconds": "542.0223661849741" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_26/boolq_2025-12-06T06-33-01.054803.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_26/boolq_2025-12-06T06-33-01.054803.json new file mode 100644 index 0000000000000000000000000000000000000000..b663c941844f49dbc4637bf5d4d7a771692fe687 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_26/boolq_2025-12-06T06-33-01.054803.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.6737003058103975, + "acc_stderr,none": 0.008200385052427135 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 32 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764973764.4926567, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1274642.732962917, + "end_time": 1274895.213172299, + "total_evaluation_time_seconds": "252.48020938201807" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_26/hellaswag_2025-12-06T07-37-23.897864.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_26/hellaswag_2025-12-06T07-37-23.897864.json new file mode 100644 index 0000000000000000000000000000000000000000..ce8e90ee66907ded823ddecf1a64fa1a8eef381a --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_26/hellaswag_2025-12-06T07-37-23.897864.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.5837482573192591, + "acc_stderr,none": 0.004919289113027502, + "acc_norm,none": 0.7955586536546505, + "acc_norm_stderr,none": 0.0040246858301306926 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 32 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764974990.5543997, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1275867.883378879, + "end_time": 1278758.056152123, + "total_evaluation_time_seconds": "2890.1727732440922" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_26/piqa_2025-12-06T06-48-26.925913.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_26/piqa_2025-12-06T06-48-26.925913.json new file mode 100644 index 0000000000000000000000000000000000000000..6ec40ac26e3e13107bdca510b176828f9c52b78a --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_26/piqa_2025-12-06T06-48-26.925913.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7894450489662677, + "acc_stderr,none": 0.009512378081238759, + "acc_norm,none": 0.7916213275299239, + "acc_norm_stderr,none": 0.009476125383049441 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764974843.4762836, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1275721.354331405, + "end_time": 1275821.083927672, + "total_evaluation_time_seconds": "99.72959626698866" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_26/truthfulqa_mc1_2025-12-06T06-46-00.585788.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_26/truthfulqa_mc1_2025-12-06T06-46-00.585788.json new file mode 100644 index 0000000000000000000000000000000000000000..2476e879e856ff46eee8db0a30a3b120534364c7 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_26/truthfulqa_mc1_2025-12-06T06-46-00.585788.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.3708690330477356, + "acc_stderr,none": 0.01690969358024884 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764974654.7257211, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1275531.26949115, + "end_time": 1275674.744129177, + "total_evaluation_time_seconds": "143.47463802690618" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_26/winogrande_2025-12-06T06-28-01.624413.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_26/winogrande_2025-12-06T06-28-01.624413.json new file mode 100644 index 0000000000000000000000000000000000000000..6334387303f751f7f5621c7332544d21c38eab02 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_26/winogrande_2025-12-06T06-28-01.624413.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.7458563535911602, + "acc_stderr,none": 0.012236307219708264 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764973612.5419068, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1274486.707488295, + "end_time": 1274595.7824074, + "total_evaluation_time_seconds": "109.07491910504177" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_27/arc_challenge_2025-12-06T08-04-38.050756.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_27/arc_challenge_2025-12-06T08-04-38.050756.json new file mode 100644 index 0000000000000000000000000000000000000000..47bc9192564fcd57ac75c69e4e53c4947032cc9a --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_27/arc_challenge_2025-12-06T08-04-38.050756.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.575938566552901, + "acc_stderr,none": 0.014441889627464401, + "acc_norm,none": 0.6040955631399317, + "acc_norm_stderr,none": 0.014291228393536585 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 32 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764978961.8304167, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1279839.029272783, + "end_time": 1280392.208794431, + "total_evaluation_time_seconds": "553.1795216479804" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_27/boolq_2025-12-06T07-54-38.225657.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_27/boolq_2025-12-06T07-54-38.225657.json new file mode 100644 index 0000000000000000000000000000000000000000..ee4625fbf10f575b94c2da3ef0784c5ba445f214 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_27/boolq_2025-12-06T07-54-38.225657.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.6935779816513762, + "acc_stderr,none": 0.008063065224064631 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 32 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764978659.6548913, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1279535.2576046, + "end_time": 1279792.383870335, + "total_evaluation_time_seconds": "257.1262657351326" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_27/hellaswag_2025-12-06T08-59-25.149275.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_27/hellaswag_2025-12-06T08-59-25.149275.json new file mode 100644 index 0000000000000000000000000000000000000000..9ed2928567f7be411b1bfd5149d5c97fb94157e3 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_27/hellaswag_2025-12-06T08-59-25.149275.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.5811591316470822, + "acc_stderr,none": 0.004923609207861548, + "acc_norm,none": 0.7938657637920733, + "acc_norm_stderr,none": 0.004037012714039217 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 32 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764979908.5671644, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1280785.268823284, + "end_time": 1283679.307605303, + "total_evaluation_time_seconds": "2894.0387820189353" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_27/piqa_2025-12-06T08-10-23.996731.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_27/piqa_2025-12-06T08-10-23.996731.json new file mode 100644 index 0000000000000000000000000000000000000000..c695082bcae262a721a7156e2a12b4fa7cd643ea --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_27/piqa_2025-12-06T08-10-23.996731.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7867247007616975, + "acc_stderr,none": 0.009557121225861324, + "acc_norm,none": 0.795429815016322, + "acc_norm_stderr,none": 0.009411688039193561 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764979757.7068086, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1280635.848213282, + "end_time": 1280738.155038273, + "total_evaluation_time_seconds": "102.30682499101385" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_27/truthfulqa_mc1_2025-12-06T08-07-54.143494.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_27/truthfulqa_mc1_2025-12-06T08-07-54.143494.json new file mode 100644 index 0000000000000000000000000000000000000000..2e6bfec2f94f10cd9ed77b9f2d0e6fbd5998a590 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_27/truthfulqa_mc1_2025-12-06T08-07-54.143494.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.3598531211750306, + "acc_stderr,none": 0.016801860466677164 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764979562.6057403, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1280438.651262257, + "end_time": 1280588.301738132, + "total_evaluation_time_seconds": "149.65047587500885" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_27/winogrande_2025-12-06T07-49-34.098533.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_27/winogrande_2025-12-06T07-49-34.098533.json new file mode 100644 index 0000000000000000000000000000000000000000..dbc3087f0db0436ad5f1e59e3e771ca02d286e06 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_27/winogrande_2025-12-06T07-49-34.098533.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.7490134175217048, + "acc_stderr,none": 0.012185776220516137 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764978502.396996, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1279376.920812261, + "end_time": 1279488.256898612, + "total_evaluation_time_seconds": "111.3360863509588" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_28/arc_challenge_2025-12-06T09-26-37.779534.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_28/arc_challenge_2025-12-06T09-26-37.779534.json new file mode 100644 index 0000000000000000000000000000000000000000..51c72c6f92046927f4865c3530fec12123593e2a --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_28/arc_challenge_2025-12-06T09-26-37.779534.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.5870307167235495, + "acc_stderr,none": 0.014388344935398326, + "acc_norm,none": 0.6186006825938567, + "acc_norm_stderr,none": 0.01419438908668526 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 32 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764983889.3731074, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1284765.834990268, + "end_time": 1285311.937604135, + "total_evaluation_time_seconds": "546.1026138667949" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_28/boolq_2025-12-06T09-16-44.874301.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_28/boolq_2025-12-06T09-16-44.874301.json new file mode 100644 index 0000000000000000000000000000000000000000..4688bfa2eefb1164329568e4128cd1eb775fa251 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_28/boolq_2025-12-06T09-16-44.874301.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.8024464831804281, + "acc_stderr,none": 0.006963746631628744 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 32 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764983583.9592087, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1284459.620381171, + "end_time": 1284719.032668981, + "total_evaluation_time_seconds": "259.41228781011887" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_28/hellaswag_2025-12-06T10-21-43.635475.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_28/hellaswag_2025-12-06T10-21-43.635475.json new file mode 100644 index 0000000000000000000000000000000000000000..c51d5c53a2e31a4e0fa86cf6c1d7a972b57bd103 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_28/hellaswag_2025-12-06T10-21-43.635475.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.5860386377215694, + "acc_stderr,none": 0.004915351107318773, + "acc_norm,none": 0.8046205935072694, + "acc_norm_stderr,none": 0.003956821705018452 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 32 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764984851.366626, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1285727.588950692, + "end_time": 1288617.793599927, + "total_evaluation_time_seconds": "2890.204649235122" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_28/piqa_2025-12-06T09-32-45.973294.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_28/piqa_2025-12-06T09-32-45.973294.json new file mode 100644 index 0000000000000000000000000000000000000000..707dfdb4137cf87797b01c1920f2f19cf6b9d2c0 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_28/piqa_2025-12-06T09-32-45.973294.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7834602829162133, + "acc_stderr,none": 0.009609984714384609, + "acc_norm,none": 0.794341675734494, + "acc_norm_stderr,none": 0.009430229076102505 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764984680.7908401, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1285559.735105248, + "end_time": 1285680.131633551, + "total_evaluation_time_seconds": "120.39652830315754" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_28/truthfulqa_mc1_2025-12-06T09-29-58.192286.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_28/truthfulqa_mc1_2025-12-06T09-29-58.192286.json new file mode 100644 index 0000000000000000000000000000000000000000..0cdd8f676cc4bf7a86427622605d6ef1685368f6 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_28/truthfulqa_mc1_2025-12-06T09-29-58.192286.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.37209302325581395, + "acc_stderr,none": 0.016921090118814038 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764984483.6799667, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1285359.142322168, + "end_time": 1285512.350650643, + "total_evaluation_time_seconds": "153.2083284750115" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_28/winogrande_2025-12-06T09-11-38.050903.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_28/winogrande_2025-12-06T09-11-38.050903.json new file mode 100644 index 0000000000000000000000000000000000000000..e13aafa02c20249f08de29544aa5bd684e41065e --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_28/winogrande_2025-12-06T09-11-38.050903.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.7458563535911602, + "acc_stderr,none": 0.012236307219708266 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764983422.5573995, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1284298.365420642, + "end_time": 1284412.209207095, + "total_evaluation_time_seconds": "113.8437864528969" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_29/arc_challenge_2025-12-06T10-48-58.487918.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_29/arc_challenge_2025-12-06T10-48-58.487918.json new file mode 100644 index 0000000000000000000000000000000000000000..ed6c103dc82af495f01e5966349f006dc1a1ff70 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_29/arc_challenge_2025-12-06T10-48-58.487918.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.5861774744027304, + "acc_stderr,none": 0.014392730009221005, + "acc_norm,none": 0.6083617747440273, + "acc_norm_stderr,none": 0.01426412212493822 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 32 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764988828.3874593, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1289706.365123845, + "end_time": 1290252.646225072, + "total_evaluation_time_seconds": "546.2811012270395" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_29/boolq_2025-12-06T10-39-05.204422.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_29/boolq_2025-12-06T10-39-05.204422.json new file mode 100644 index 0000000000000000000000000000000000000000..631adbeefef6cf2e1a60ebbe09ea44331f585ccd --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_29/boolq_2025-12-06T10-39-05.204422.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.710703363914373, + "acc_stderr,none": 0.007930643304471017 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 32 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764988523.2568605, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1289400.793326064, + "end_time": 1289659.362734898, + "total_evaluation_time_seconds": "258.56940883398056" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_29/hellaswag_2025-12-06T11-44-01.108602.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_29/hellaswag_2025-12-06T11-44-01.108602.json new file mode 100644 index 0000000000000000000000000000000000000000..7b9682b19960f522d95510296c3864940206efd0 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_29/hellaswag_2025-12-06T11-44-01.108602.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.5825532762397929, + "acc_stderr,none": 0.004921300331285549, + "acc_norm,none": 0.799044015136427, + "acc_norm_stderr,none": 0.003998962580974764 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 32 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764989788.8325799, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1290666.380968803, + "end_time": 1293555.266710426, + "total_evaluation_time_seconds": "2888.8857416228857" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_29/piqa_2025-12-06T10-55-06.017232.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_29/piqa_2025-12-06T10-55-06.017232.json new file mode 100644 index 0000000000000000000000000000000000000000..ee8dafeceb6823a21dc334da779d18b4c4c473bc --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_29/piqa_2025-12-06T10-55-06.017232.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7894450489662677, + "acc_stderr,none": 0.00951237808123876, + "acc_norm,none": 0.7932535364526659, + "acc_norm_stderr,none": 0.009448665514183267 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764989638.428643, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1290516.565041746, + "end_time": 1290620.175260238, + "total_evaluation_time_seconds": "103.61021849187091" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_3/arc_challenge_2025-12-04T23-14-42.807847.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_3/arc_challenge_2025-12-04T23-14-42.807847.json new file mode 100644 index 0000000000000000000000000000000000000000..7c7c9a31aae29b85639ce4a8d5622592b5dcd157 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_3/arc_challenge_2025-12-04T23-14-42.807847.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.5315699658703071, + "acc_stderr,none": 0.014582236460866973, + "acc_norm,none": 0.5614334470989761, + "acc_norm_stderr,none": 0.014500682618212865 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 32 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764860763.477095, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1161644.164346931, + "end_time": 1162196.965976054, + "total_evaluation_time_seconds": "552.8016291230451" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_3/boolq_2025-12-04T23-04-35.499288.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_3/boolq_2025-12-04T23-04-35.499288.json new file mode 100644 index 0000000000000000000000000000000000000000..e0ed5eb426bd58007577e56750f5f36fffadd333 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_3/boolq_2025-12-04T23-04-35.499288.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.4960244648318043, + "acc_stderr,none": 0.008744778542942208 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 32 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764860440.9313679, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1161316.335959531, + "end_time": 1161589.657326559, + "total_evaluation_time_seconds": "273.32136702816933" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_3/truthfulqa_mc1_2025-12-04T23-18-15.633405.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_3/truthfulqa_mc1_2025-12-04T23-18-15.633405.json new file mode 100644 index 0000000000000000000000000000000000000000..a205da5ecb89179cdf882cfad8d4ffeba5b20c49 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_3/truthfulqa_mc1_2025-12-04T23-18-15.633405.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.32068543451652387, + "acc_stderr,none": 0.016339170373280906 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764861370.3295672, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1162245.000239355, + "end_time": 1162409.791748103, + "total_evaluation_time_seconds": "164.79150874796323" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_3/winogrande_2025-12-04T22-59-14.704300.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_3/winogrande_2025-12-04T22-59-14.704300.json new file mode 100644 index 0000000000000000000000000000000000000000..f3520ea3ff93ba68a40014e5be6df343ca16cb70 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_3/winogrande_2025-12-04T22-59-14.704300.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.5966850828729282, + "acc_stderr,none": 0.013787257285896243 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764860270.5147939, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1161149.937722449, + "end_time": 1161268.862613558, + "total_evaluation_time_seconds": "118.92489110911265" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_30/arc_challenge_2025-12-06T12-11-18.144793.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_30/arc_challenge_2025-12-06T12-11-18.144793.json new file mode 100644 index 0000000000000000000000000000000000000000..52ed01fa0a6b2abe20f493ed3005dcc7d879fb81 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_30/arc_challenge_2025-12-06T12-11-18.144793.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.5674061433447098, + "acc_stderr,none": 0.014478005694182526, + "acc_norm,none": 0.6023890784982935, + "acc_norm_stderr,none": 0.014301752223279536 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 32 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764993768.5614245, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1294645.016185531, + "end_time": 1295192.302903132, + "total_evaluation_time_seconds": "547.2867176011205" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_30/hellaswag_2025-12-06T13-05-58.601406.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_30/hellaswag_2025-12-06T13-05-58.601406.json new file mode 100644 index 0000000000000000000000000000000000000000..068b6ccff9bcd4cc6b6abc2262497318e7bcb9e8 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_30/hellaswag_2025-12-06T13-05-58.601406.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.5814578769169488, + "acc_stderr,none": 0.004923117849740296, + "acc_norm,none": 0.7977494523003386, + "acc_norm_stderr,none": 0.00400857143148369 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 32 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764994707.1249542, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1295583.622431367, + "end_time": 1298472.759776034, + "total_evaluation_time_seconds": "2889.1373446669895" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_30/truthfulqa_mc1_2025-12-06T12-14-35.273914.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_30/truthfulqa_mc1_2025-12-06T12-14-35.273914.json new file mode 100644 index 0000000000000000000000000000000000000000..03bbdbc44b95a0daf475fa02e7066c2e68f618fd --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_30/truthfulqa_mc1_2025-12-06T12-14-35.273914.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.39167686658506734, + "acc_stderr,none": 0.01708779588176965 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764994365.0639431, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1295239.915218326, + "end_time": 1295389.432172153, + "total_evaluation_time_seconds": "149.51695382688195" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_30/winogrande_2025-12-06T11-56-17.498824.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_30/winogrande_2025-12-06T11-56-17.498824.json new file mode 100644 index 0000000000000000000000000000000000000000..5c63bf687488e30c6cef31f5224c2c314ab4d3ef --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_30/winogrande_2025-12-06T11-56-17.498824.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.7505919494869772, + "acc_stderr,none": 0.012160189196930687 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764993297.9188924, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1294172.573700307, + "end_time": 1294291.65720898, + "total_evaluation_time_seconds": "119.08350867312402" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_31/piqa_2025-12-06T13-39-36.615922.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_31/piqa_2025-12-06T13-39-36.615922.json new file mode 100644 index 0000000000000000000000000000000000000000..4995c600ee19af9415640e40458b7541abafcac6 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_31/piqa_2025-12-06T13-39-36.615922.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7823721436343852, + "acc_stderr,none": 0.009627407474840878, + "acc_norm,none": 0.7932535364526659, + "acc_norm_stderr,none": 0.00944866551418327 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764999505.6579092, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1300384.551678243, + "end_time": 1300490.774189221, + "total_evaluation_time_seconds": "106.22251097788103" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_32/boolq_2025-12-08T18-14-39.362228.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_32/boolq_2025-12-08T18-14-39.362228.json new file mode 100644 index 0000000000000000000000000000000000000000..0637e158494c3373bd52e5c6c0744066faec7b21 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_32/boolq_2025-12-08T18-14-39.362228.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7712538226299694, + "acc_stderr,none": 0.007346290992436894 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 32 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1765188655.3549364, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1489530.496495215, + "end_time": 1489793.520593146, + "total_evaluation_time_seconds": "263.02409793110564" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_34/piqa_2025-12-08T21-17-17.424864.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_34/piqa_2025-12-08T21-17-17.424864.json new file mode 100644 index 0000000000000000000000000000000000000000..4c9bda6f1809cadf85aac9d422fd74b85b771051 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_34/piqa_2025-12-08T21-17-17.424864.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7856365614798694, + "acc_stderr,none": 0.009574842136050947, + "acc_norm,none": 0.7916213275299239, + "acc_norm_stderr,none": 0.009476125383049452 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1765199760.7186577, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1500636.180086612, + "end_time": 1500751.583184187, + "total_evaluation_time_seconds": "115.40309757506475" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_34/winogrande_2025-12-08T20-55-48.817018.json b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_34/winogrande_2025-12-08T20-55-48.817018.json new file mode 100644 index 0000000000000000000000000000000000000000..56440dba16da81c72c580d6035f6078f8a163d4d --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-14B-quantization-layer-mlp/mlp_34/winogrande_2025-12-08T20-55-48.817018.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.755327545382794, + "acc_stderr,none": 0.012082125654159738 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_num_parameters": 14770033664, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1765198468.4211493, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-14B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-14B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1499343.819570472, + "end_time": 1499462.975275677, + "total_evaluation_time_seconds": "119.15570520493202" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-fg1/fg1/piqa_2025-12-06T03-16-06.550402.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-fg1/fg1/piqa_2025-12-06T03-16-06.550402.json new file mode 100644 index 0000000000000000000000000000000000000000..dabe9aa6401497d9fe8ff722ceafb638363e8a47 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-fg1/fg1/piqa_2025-12-06T03-16-06.550402.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.6523394994559304, + "acc_stderr,none": 0.011111173661393732, + "acc_norm,none": 0.6566920565832427, + "acc_norm_stderr,none": 0.011078175207348842 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-fg" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-fg", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764962120.6874478, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-fg", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-fg", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1262999.497138259, + "end_time": 1263080.708677228, + "total_evaluation_time_seconds": "81.21153896907344" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-fg1/fg1/truthfulqa_mc1_2025-12-06T03-13-58.537078.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-fg1/fg1/truthfulqa_mc1_2025-12-06T03-13-58.537078.json new file mode 100644 index 0000000000000000000000000000000000000000..2d794f0d766442d7436934b66b8efe2b186126c5 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-fg1/fg1/truthfulqa_mc1_2025-12-06T03-13-58.537078.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.2802937576499388, + "acc_stderr,none": 0.015723139524608763 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-fg" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-fg", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764961928.4049907, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-fg", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-fg", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1262806.145706363, + "end_time": 1262952.695082166, + "total_evaluation_time_seconds": "146.549375802977" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-fg1/fg1/winogrande_2025-12-05T18-05-37.805389.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-fg1/fg1/winogrande_2025-12-05T18-05-37.805389.json new file mode 100644 index 0000000000000000000000000000000000000000..0a6b7241e06eeeb29f5522f9c78823d2e4227954 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-fg1/fg1/winogrande_2025-12-05T18-05-37.805389.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.4996053670086819, + "acc_stderr,none": 0.014052481306049512 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-fg" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-fg", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764929071.40178, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-fg", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-fg", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1229953.673254053, + "end_time": 1230051.96372227, + "total_evaluation_time_seconds": "98.2904682168737" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_-1/hellaswag_2025-12-04T02-12-46.339420.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_-1/hellaswag_2025-12-04T02-12-46.339420.json new file mode 100644 index 0000000000000000000000000000000000000000..854dd341ebf51c2642751de8a463c52c33e3e57a --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_-1/hellaswag_2025-12-04T02-12-46.339420.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.5102569209320852, + "acc_stderr,none": 0.004988731406780666, + "acc_norm,none": 0.717486556462856, + "acc_norm_stderr,none": 0.00449301594559971 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764782841.126278, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1083724.108217528, + "end_time": 1086480.497765565, + "total_evaluation_time_seconds": "2756.3895480369683" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_0/boolq_2025-12-04T02-22-53.291249.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_0/boolq_2025-12-04T02-22-53.291249.json new file mode 100644 index 0000000000000000000000000000000000000000..5d3fbe02165845bda8099b30dca2f15df03aff5d --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_0/boolq_2025-12-04T02-22-53.291249.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.6657492354740061, + "acc_stderr,none": 0.008250572455083427 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764786050.064723, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1086932.357094647, + "end_time": 1087087.449564896, + "total_evaluation_time_seconds": "155.09247024916112" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_0/hellaswag_2025-12-04T03-21-26.068666.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_0/hellaswag_2025-12-04T03-21-26.068666.json new file mode 100644 index 0000000000000000000000000000000000000000..519941334fe3519760aaf969e82a5d7c9e33ba83 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_0/hellaswag_2025-12-04T03-21-26.068666.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.35540728938458477, + "acc_stderr,none": 0.004776583530909575, + "acc_norm,none": 0.4643497311292571, + "acc_norm_stderr,none": 0.004977081808179436 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764786966.7121048, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1087850.192416947, + "end_time": 1090600.226907812, + "total_evaluation_time_seconds": "2750.0344908649568" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_1/arc_challenge_2025-12-04T03-39-22.216157.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_1/arc_challenge_2025-12-04T03-39-22.216157.json new file mode 100644 index 0000000000000000000000000000000000000000..28eafce3e29633af89a2ea73e37f363f1a98b17c --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_1/arc_challenge_2025-12-04T03-39-22.216157.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.2150170648464164, + "acc_stderr,none": 0.012005717634133611, + "acc_norm,none": 0.26706484641638223, + "acc_norm_stderr,none": 0.012928933196496354 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764790334.430762, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1091216.82649347, + "end_time": 1091676.374211946, + "total_evaluation_time_seconds": "459.5477184760384" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_1/hellaswag_2025-12-04T04-29-28.197024.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_1/hellaswag_2025-12-04T04-29-28.197024.json new file mode 100644 index 0000000000000000000000000000000000000000..051a52c7b9274db3f5e68e8fa6f98fbbef4dfac8 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_1/hellaswag_2025-12-04T04-29-28.197024.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.2593108942441745, + "acc_stderr,none": 0.004373608212561033, + "acc_norm,none": 0.25871340370444135, + "acc_norm_stderr,none": 0.004370328224831799 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764791069.2130196, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1091951.719263049, + "end_time": 1094682.355289199, + "total_evaluation_time_seconds": "2730.6360261498485" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_10/boolq_2025-12-04T13-50-18.379159.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_10/boolq_2025-12-04T13-50-18.379159.json new file mode 100644 index 0000000000000000000000000000000000000000..613b6c1f1f5c4126f6f33899d6b48f49539c40ca --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_10/boolq_2025-12-04T13-50-18.379159.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7162079510703364, + "acc_stderr,none": 0.007885191054174696 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764827292.5150957, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1128171.668460108, + "end_time": 1128332.537450986, + "total_evaluation_time_seconds": "160.86899087787606" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_10/hellaswag_2025-12-04T14-49-17.262148.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_10/hellaswag_2025-12-04T14-49-17.262148.json new file mode 100644 index 0000000000000000000000000000000000000000..c669c6a374308312d0e91562a5900918e3043fa0 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_10/hellaswag_2025-12-04T14-49-17.262148.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.47629954192391954, + "acc_stderr,none": 0.00498417262182288, + "acc_norm,none": 0.6752638916550487, + "acc_norm_stderr,none": 0.0046731914238612405 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764828218.9367275, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1129101.119130078, + "end_time": 1131871.420485455, + "total_evaluation_time_seconds": "2770.3013553770725" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_10/piqa_2025-12-04T14-02-39.610881.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_10/piqa_2025-12-04T14-02-39.610881.json new file mode 100644 index 0000000000000000000000000000000000000000..c5d47ba1f499ff39c6f915054df42a7cab132043 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_10/piqa_2025-12-04T14-02-39.610881.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7377584330794341, + "acc_stderr,none": 0.010262502565172443, + "acc_norm,none": 0.7410228509249184, + "acc_norm_stderr,none": 0.010220966031405604 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764828113.37022, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1128996.700050376, + "end_time": 1129073.76918529, + "total_evaluation_time_seconds": "77.0691349140834" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_10/truthfulqa_mc1_2025-12-04T14-00-56.457379.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_10/truthfulqa_mc1_2025-12-04T14-00-56.457379.json new file mode 100644 index 0000000000000000000000000000000000000000..0b21fe764bc2667ffaf5f7f93d011b7b75588a62 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_10/truthfulqa_mc1_2025-12-04T14-00-56.457379.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.2631578947368421, + "acc_stderr,none": 0.01541524174023702 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764827964.8817825, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1128847.364514757, + "end_time": 1128970.615700455, + "total_evaluation_time_seconds": "123.25118569796905" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_10/winogrande_2025-12-04T13-47-08.327929.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_10/winogrande_2025-12-04T13-47-08.327929.json new file mode 100644 index 0000000000000000000000000000000000000000..e2dd5282b9f14918efb4b22c2a3b5f825724bc78 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_10/winogrande_2025-12-04T13-47-08.327929.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.5422257300710339, + "acc_stderr,none": 0.014002284504422433 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764827169.5049548, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1128052.272212956, + "end_time": 1128142.484922688, + "total_evaluation_time_seconds": "90.21270973188803" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_11/arc_challenge_2025-12-04T15-07-49.164052.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_11/arc_challenge_2025-12-04T15-07-49.164052.json new file mode 100644 index 0000000000000000000000000000000000000000..872b77527378aaebd4f8c3500a8b2fb85ebebaed --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_11/arc_challenge_2025-12-04T15-07-49.164052.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.4539249146757679, + "acc_stderr,none": 0.014549221105171864, + "acc_norm,none": 0.514505119453925, + "acc_norm_stderr,none": 0.014605241081370056 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764831639.5029724, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1132521.073728188, + "end_time": 1132983.32239084, + "total_evaluation_time_seconds": "462.24866265198216" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_11/boolq_2025-12-04T14-59-39.379497.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_11/boolq_2025-12-04T14-59-39.379497.json new file mode 100644 index 0000000000000000000000000000000000000000..e46262f3eecb1bb6b124b136a2c2c2106984e691 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_11/boolq_2025-12-04T14-59-39.379497.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7428134556574923, + "acc_stderr,none": 0.007644631266348265 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764831449.4460344, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1132329.727129936, + "end_time": 1132493.537785202, + "total_evaluation_time_seconds": "163.81065526604652" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_11/piqa_2025-12-04T15-12-18.165918.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_11/piqa_2025-12-04T15-12-18.165918.json new file mode 100644 index 0000000000000000000000000000000000000000..968179285384f7c571311ca71a040b0ddba49f64 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_11/piqa_2025-12-04T15-12-18.165918.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.749727965179543, + "acc_stderr,none": 0.010106561880089801, + "acc_norm,none": 0.7665941240478781, + "acc_norm_stderr,none": 0.009869247889521015 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764832284.975282, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1133167.3383595, + "end_time": 1133252.324263055, + "total_evaluation_time_seconds": "84.98590355506167" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_11/truthfulqa_mc1_2025-12-04T15-10-25.100852.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_11/truthfulqa_mc1_2025-12-04T15-10-25.100852.json new file mode 100644 index 0000000000000000000000000000000000000000..41696070fe8221f53add4ca3a4d497a68107df4e --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_11/truthfulqa_mc1_2025-12-04T15-10-25.100852.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.3292533659730722, + "acc_stderr,none": 0.016451264440068232 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764832130.019068, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1133012.110083649, + "end_time": 1133139.259121703, + "total_evaluation_time_seconds": "127.14903805404902" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_11/winogrande_2025-12-04T14-56-28.307523.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_11/winogrande_2025-12-04T14-56-28.307523.json new file mode 100644 index 0000000000000000000000000000000000000000..2d73c447a61d7268d21c5f2998d36656c368df1d --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_11/winogrande_2025-12-04T14-56-28.307523.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.580110497237569, + "acc_stderr,none": 0.013870943986310391 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764831321.5065958, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1132204.325634179, + "end_time": 1132302.464966432, + "total_evaluation_time_seconds": "98.13933225302026" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_12/arc_challenge_2025-12-04T16-17-06.443939.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_12/arc_challenge_2025-12-04T16-17-06.443939.json new file mode 100644 index 0000000000000000000000000000000000000000..55668decf8185e1b28881ec96bcf7f026720e189 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_12/arc_challenge_2025-12-04T16-17-06.443939.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.4641638225255973, + "acc_stderr,none": 0.014573813664735718, + "acc_norm,none": 0.5170648464163823, + "acc_norm_stderr,none": 0.014602878388536593 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764835800.0427787, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1136682.877291522, + "end_time": 1137140.602073627, + "total_evaluation_time_seconds": "457.7247821050696" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_12/boolq_2025-12-04T16-09-01.992682.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_12/boolq_2025-12-04T16-09-01.992682.json new file mode 100644 index 0000000000000000000000000000000000000000..a648a4408f9f2e94cb268e9d8b552bc978a73508 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_12/boolq_2025-12-04T16-09-01.992682.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.6914373088685015, + "acc_stderr,none": 0.00807868456602172 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764835613.481713, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1136494.455756693, + "end_time": 1136656.151032966, + "total_evaluation_time_seconds": "161.69527627318166" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_12/hellaswag_2025-12-04T17-08-06.569358.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_12/hellaswag_2025-12-04T17-08-06.569358.json new file mode 100644 index 0000000000000000000000000000000000000000..ed4901c8a856b949360d10f3dd2aa6133ae9b651 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_12/hellaswag_2025-12-04T17-08-06.569358.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.4927305317665804, + "acc_stderr,none": 0.004989254011895762, + "acc_norm,none": 0.6942840071698865, + "acc_norm_stderr,none": 0.004597684609707813 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764836546.5279205, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1137428.975899189, + "end_time": 1140200.727386846, + "total_evaluation_time_seconds": "2771.751487656962" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_12/piqa_2025-12-04T16-21-26.120280.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_12/piqa_2025-12-04T16-21-26.120280.json new file mode 100644 index 0000000000000000000000000000000000000000..f1d097ce8b2196886f6a9b79317b215a14206c7d --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_12/piqa_2025-12-04T16-21-26.120280.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7415669205658324, + "acc_stderr,none": 0.010213971636773336, + "acc_norm,none": 0.7584330794341676, + "acc_norm_stderr,none": 0.009986718001804479 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764836434.444799, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1137317.156486259, + "end_time": 1137400.278598195, + "total_evaluation_time_seconds": "83.122111935867" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_12/truthfulqa_mc1_2025-12-04T16-19-37.225420.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_12/truthfulqa_mc1_2025-12-04T16-19-37.225420.json new file mode 100644 index 0000000000000000000000000000000000000000..7fd4edf5dc1913bc39d4451f58f956c5e1de2d7a --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_12/truthfulqa_mc1_2025-12-04T16-19-37.225420.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.31946144430844553, + "acc_stderr,none": 0.0163226441829605 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764836285.958567, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1137168.775813601, + "end_time": 1137291.383768981, + "total_evaluation_time_seconds": "122.60795538011007" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_12/winogrande_2025-12-04T16-05-51.702649.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_12/winogrande_2025-12-04T16-05-51.702649.json new file mode 100644 index 0000000000000000000000000000000000000000..782c1a411b6c5db0c98477a92b22c90ef1e37d53 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_12/winogrande_2025-12-04T16-05-51.702649.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.5982636148382005, + "acc_stderr,none": 0.01377843926664949 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764835488.8255577, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1136371.018582831, + "end_time": 1136465.860956152, + "total_evaluation_time_seconds": "94.84237332106568" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_13/arc_challenge_2025-12-04T17-26-32.719356.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_13/arc_challenge_2025-12-04T17-26-32.719356.json new file mode 100644 index 0000000000000000000000000000000000000000..f43d6cfe179d729a2696f65a22e3ad49de00a9bc --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_13/arc_challenge_2025-12-04T17-26-32.719356.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.4667235494880546, + "acc_stderr,none": 0.014578995859605816, + "acc_norm,none": 0.5179180887372014, + "acc_norm_stderr,none": 0.014602005585490978 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764839965.2306778, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1140846.50715149, + "end_time": 1141306.877661194, + "total_evaluation_time_seconds": "460.37050970387645" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_13/boolq_2025-12-04T17-18-22.898564.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_13/boolq_2025-12-04T17-18-22.898564.json new file mode 100644 index 0000000000000000000000000000000000000000..5fada5cd16de0b6512c95156bebe05773ba77d8a --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_13/boolq_2025-12-04T17-18-22.898564.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.6929663608562691, + "acc_stderr,none": 0.008067548709328671 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764839775.7153027, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1140656.574947321, + "end_time": 1140817.056896256, + "total_evaluation_time_seconds": "160.4819489349611" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_13/hellaswag_2025-12-04T18-17-19.530603.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_13/hellaswag_2025-12-04T18-17-19.530603.json new file mode 100644 index 0000000000000000000000000000000000000000..1280bb82c8e887ccd915f12439dfc5a9ca1990d6 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_13/hellaswag_2025-12-04T18-17-19.530603.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.4894443337980482, + "acc_stderr,none": 0.004988669343786964, + "acc_norm,none": 0.691396136227843, + "acc_norm_stderr,none": 0.00460973192573695 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764840713.5220544, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1141595.57063991, + "end_time": 1144353.688672265, + "total_evaluation_time_seconds": "2758.1180323550943" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_13/piqa_2025-12-04T17-30-52.493279.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_13/piqa_2025-12-04T17-30-52.493279.json new file mode 100644 index 0000000000000000000000000000000000000000..f9ce5fa157976fcb757cb0144452063fd0cb6d7c --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_13/piqa_2025-12-04T17-30-52.493279.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.749727965179543, + "acc_stderr,none": 0.010106561880089798, + "acc_norm,none": 0.7616974972796517, + "acc_norm_stderr,none": 0.009940334245876209 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764840605.5129879, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1141487.82901299, + "end_time": 1141566.651523276, + "total_evaluation_time_seconds": "78.82251028600149" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_13/truthfulqa_mc1_2025-12-04T17-29-06.089516.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_13/truthfulqa_mc1_2025-12-04T17-29-06.089516.json new file mode 100644 index 0000000000000000000000000000000000000000..cd3e4f9434be045213fc12c35f8c1174e2adb1da --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_13/truthfulqa_mc1_2025-12-04T17-29-06.089516.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.3047735618115055, + "acc_stderr,none": 0.016114124156882438 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764840450.4012074, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1141333.21603554, + "end_time": 1141460.247547168, + "total_evaluation_time_seconds": "127.03151162806898" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_13/winogrande_2025-12-04T17-15-09.573331.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_13/winogrande_2025-12-04T17-15-09.573331.json new file mode 100644 index 0000000000000000000000000000000000000000..90a7bb59273fa7aef6df56631d8eafada8039c68 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_13/winogrande_2025-12-04T17-15-09.573331.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.5864246250986582, + "acc_stderr,none": 0.013840971763195296 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764839650.7221138, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1140532.482258339, + "end_time": 1140623.730507701, + "total_evaluation_time_seconds": "91.24824936199002" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_14/arc_challenge_2025-12-04T18-35-45.329495.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_14/arc_challenge_2025-12-04T18-35-45.329495.json new file mode 100644 index 0000000000000000000000000000000000000000..0a2955b307d06f3277ddd92de4fa7b54cc954b92 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_14/arc_challenge_2025-12-04T18-35-45.329495.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.4880546075085324, + "acc_stderr,none": 0.014607220340597167, + "acc_norm,none": 0.5477815699658704, + "acc_norm_stderr,none": 0.014544519880633827 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764844116.5711956, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1144997.697990533, + "end_time": 1145459.487813439, + "total_evaluation_time_seconds": "461.7898229060229" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_14/boolq_2025-12-04T18-27-35.505360.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_14/boolq_2025-12-04T18-27-35.505360.json new file mode 100644 index 0000000000000000000000000000000000000000..f074cd714036b32953ea5e9a4f052f0920349ca9 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_14/boolq_2025-12-04T18-27-35.505360.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7201834862385321, + "acc_stderr,none": 0.00785146664100134 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764843928.7080681, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1144811.038104945, + "end_time": 1144969.663671521, + "total_evaluation_time_seconds": "158.62556657590903" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_14/hellaswag_2025-12-04T19-26-58.769218.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_14/hellaswag_2025-12-04T19-26-58.769218.json new file mode 100644 index 0000000000000000000000000000000000000000..7470d08ddf1ced3b48ee7ddd4a5360bab38045fc --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_14/hellaswag_2025-12-04T19-26-58.769218.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.49751045608444533, + "acc_stderr,none": 0.004989719559439897, + "acc_norm,none": 0.7022505476996614, + "acc_norm_stderr,none": 0.004563342285538032 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764844888.2737298, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1145771.151232913, + "end_time": 1148532.927559906, + "total_evaluation_time_seconds": "2761.7763269930147" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_14/piqa_2025-12-04T18-40-29.161230.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_14/piqa_2025-12-04T18-40-29.161230.json new file mode 100644 index 0000000000000000000000000000000000000000..bc962a03872e1c64e0613b16de5a5b22c877354d --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_14/piqa_2025-12-04T18-40-29.161230.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7519042437431991, + "acc_stderr,none": 0.010077118315574713, + "acc_norm,none": 0.7676822633297062, + "acc_norm_stderr,none": 0.009853201384168243 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764844775.1293745, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1145657.611304999, + "end_time": 1145743.319471493, + "total_evaluation_time_seconds": "85.7081664938014" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_14/truthfulqa_mc1_2025-12-04T18-38-35.993666.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_14/truthfulqa_mc1_2025-12-04T18-38-35.993666.json new file mode 100644 index 0000000000000000000000000000000000000000..ce8c7e9c8c1cfb05eb6eb1b7648556490c0c9df5 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_14/truthfulqa_mc1_2025-12-04T18-38-35.993666.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.33047735618115054, + "acc_stderr,none": 0.01646676961369831 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764844608.1693027, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1145488.989722017, + "end_time": 1145630.151869607, + "total_evaluation_time_seconds": "141.1621475899592" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_14/winogrande_2025-12-04T18-24-27.525779.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_14/winogrande_2025-12-04T18-24-27.525779.json new file mode 100644 index 0000000000000000000000000000000000000000..19e98d991edb2decf0df54af6e0b4981deebea72 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_14/winogrande_2025-12-04T18-24-27.525779.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.5785319652722968, + "acc_stderr,none": 0.013878072377497603 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764843803.6463962, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1144686.142478484, + "end_time": 1144781.683931831, + "total_evaluation_time_seconds": "95.54145334707573" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_15/arc_challenge_2025-12-04T19-46-13.374859.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_15/arc_challenge_2025-12-04T19-46-13.374859.json new file mode 100644 index 0000000000000000000000000000000000000000..26de7bfa6c5578f69b7f8723d58144870ff3a1cd --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_15/arc_challenge_2025-12-04T19-46-13.374859.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.4906143344709898, + "acc_stderr,none": 0.014608816322065003, + "acc_norm,none": 0.5213310580204779, + "acc_norm_stderr,none": 0.014598087973127104 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764848334.2870848, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1149215.258855077, + "end_time": 1149687.533079209, + "total_evaluation_time_seconds": "472.2742241320666" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_15/boolq_2025-12-04T19-37-50.545878.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_15/boolq_2025-12-04T19-37-50.545878.json new file mode 100644 index 0000000000000000000000000000000000000000..29137e77b1a604b25bb6e1441d21578dd34d761b --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_15/boolq_2025-12-04T19-37-50.545878.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.72782874617737, + "acc_stderr,none": 0.007784456012070833 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764848111.9985654, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1148994.05008305, + "end_time": 1149184.702939066, + "total_evaluation_time_seconds": "190.65285601606593" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_15/hellaswag_2025-12-04T20-37-36.589978.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_15/hellaswag_2025-12-04T20-37-36.589978.json new file mode 100644 index 0000000000000000000000000000000000000000..60dbf99e21e7c242c4d57ec542977f6f99dbe0de --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_15/hellaswag_2025-12-04T20-37-36.589978.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.4963154750049791, + "acc_stderr,none": 0.0049896459298114405, + "acc_norm,none": 0.7006572395937064, + "acc_norm_stderr,none": 0.004570342034463221 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764849119.3574395, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1150001.669406829, + "end_time": 1152770.748217249, + "total_evaluation_time_seconds": "2769.07881042012" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_15/piqa_2025-12-04T19-50-59.135125.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_15/piqa_2025-12-04T19-50-59.135125.json new file mode 100644 index 0000000000000000000000000000000000000000..8c097e1d2474ea5f07befe9019ae4557ebbff614 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_15/piqa_2025-12-04T19-50-59.135125.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7442872687704026, + "acc_stderr,none": 0.010178690109459864, + "acc_norm,none": 0.750816104461371, + "acc_norm_stderr,none": 0.010091882770120214 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764848999.276233, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1149880.839844012, + "end_time": 1149973.292971541, + "total_evaluation_time_seconds": "92.45312752900645" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_15/truthfulqa_mc1_2025-12-04T19-48-58.110298.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_15/truthfulqa_mc1_2025-12-04T19-48-58.110298.json new file mode 100644 index 0000000000000000000000000000000000000000..3d8ec9108fd9c2c324adeff9b20d7a08f347772d --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_15/truthfulqa_mc1_2025-12-04T19-48-58.110298.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.2913096695226438, + "acc_stderr,none": 0.015905987048184828 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764848834.1504343, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1149716.012416284, + "end_time": 1149852.268366769, + "total_evaluation_time_seconds": "136.2559504848905" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_15/winogrande_2025-12-04T19-34-11.503019.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_15/winogrande_2025-12-04T19-34-11.503019.json new file mode 100644 index 0000000000000000000000000000000000000000..86cc94a7ea446c5f0d44d97a4f9bbbc548d8d7b4 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_15/winogrande_2025-12-04T19-34-11.503019.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.5816890292028414, + "acc_stderr,none": 0.013863669961195885 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764847982.2835627, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1148864.172560509, + "end_time": 1148965.660082001, + "total_evaluation_time_seconds": "101.48752149194479" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_16/arc_challenge_2025-12-04T20-56-36.860832.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_16/arc_challenge_2025-12-04T20-56-36.860832.json new file mode 100644 index 0000000000000000000000000000000000000000..b70cbc436a3226d2fdce8b53a3f353a23fb5ba9e --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_16/arc_challenge_2025-12-04T20-56-36.860832.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.5068259385665529, + "acc_stderr,none": 0.014610029151379813, + "acc_norm,none": 0.5426621160409556, + "acc_norm_stderr,none": 0.014558106543924067 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764852554.6339426, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1153436.007270752, + "end_time": 1153911.018780268, + "total_evaluation_time_seconds": "475.0115095160436" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_16/boolq_2025-12-04T20-48-13.618105.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_16/boolq_2025-12-04T20-48-13.618105.json new file mode 100644 index 0000000000000000000000000000000000000000..e7ecd2b6760cad15d1427b7ac2b8dedb114bf1bb --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_16/boolq_2025-12-04T20-48-13.618105.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7535168195718654, + "acc_stderr,none": 0.007537592585606517 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764852357.3462913, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1153239.174123761, + "end_time": 1153407.775275495, + "total_evaluation_time_seconds": "168.60115173389204" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_16/hellaswag_2025-12-04T21-48-08.775777.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_16/hellaswag_2025-12-04T21-48-08.775777.json new file mode 100644 index 0000000000000000000000000000000000000000..1c45ff8883a2978431d7802ae3f405db7018d283 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_16/hellaswag_2025-12-04T21-48-08.775777.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.49263095000995816, + "acc_stderr,none": 0.004989239462835219, + "acc_norm,none": 0.6969727145986855, + "acc_norm_stderr,none": 0.004586276903267021 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764853338.627664, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1154220.593251484, + "end_time": 1157002.934090281, + "total_evaluation_time_seconds": "2782.340838796925" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_16/piqa_2025-12-04T21-01-17.983190.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_16/piqa_2025-12-04T21-01-17.983190.json new file mode 100644 index 0000000000000000000000000000000000000000..ffd8449d81dacb9a775c8d1916262f972bfc5b5a --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_16/piqa_2025-12-04T21-01-17.983190.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7535364526659413, + "acc_stderr,none": 0.010054810789671817, + "acc_norm,none": 0.7704026115342764, + "acc_norm_stderr,none": 0.009812682950815192 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764853222.9640827, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1154104.778658196, + "end_time": 1154192.141442929, + "total_evaluation_time_seconds": "87.36278473283164" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_16/truthfulqa_mc1_2025-12-04T20-59-22.015004.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_16/truthfulqa_mc1_2025-12-04T20-59-22.015004.json new file mode 100644 index 0000000000000000000000000000000000000000..a5d32f74fffbe19d509a142b290e9c3507a09dfd --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_16/truthfulqa_mc1_2025-12-04T20-59-22.015004.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.3011015911872705, + "acc_stderr,none": 0.016058999026100595 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764853058.6306806, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1153939.577979844, + "end_time": 1154076.172840252, + "total_evaluation_time_seconds": "136.59486040798947" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_16/winogrande_2025-12-04T20-44-57.834560.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_16/winogrande_2025-12-04T20-44-57.834560.json new file mode 100644 index 0000000000000000000000000000000000000000..f8cb622a4b1cc3065aa2b44813bd4a4f8aebb4a4 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_16/winogrande_2025-12-04T20-44-57.834560.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.5919494869771112, + "acc_stderr,none": 0.013812822643745034 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764852222.2261748, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1153103.25399914, + "end_time": 1153211.992837687, + "total_evaluation_time_seconds": "108.73883854690939" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_17/arc_challenge_2025-12-04T22-06-35.431206.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_17/arc_challenge_2025-12-04T22-06-35.431206.json new file mode 100644 index 0000000000000000000000000000000000000000..85979ad780b7e77c1ac0e9d01b9d4ce37be6cb69 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_17/arc_challenge_2025-12-04T22-06-35.431206.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.4974402730375427, + "acc_stderr,none": 0.014611199329843774, + "acc_norm,none": 0.5401023890784983, + "acc_norm_stderr,none": 0.014564318856924848 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764856763.3221328, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1157644.540066245, + "end_time": 1158109.589528567, + "total_evaluation_time_seconds": "465.04946232191287" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_17/boolq_2025-12-04T21-58-23.048499.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_17/boolq_2025-12-04T21-58-23.048499.json new file mode 100644 index 0000000000000000000000000000000000000000..87100755eac46fb8bf61b5bc64871780c4da76b8 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_17/boolq_2025-12-04T21-58-23.048499.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7412844036697248, + "acc_stderr,none": 0.0076594269107636975 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764856572.6471887, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1157454.91512253, + "end_time": 1157617.206739641, + "total_evaluation_time_seconds": "162.29161711107008" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_17/hellaswag_2025-12-04T22-57-38.905709.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_17/hellaswag_2025-12-04T22-57-38.905709.json new file mode 100644 index 0000000000000000000000000000000000000000..dbe3103ab620ae21e0bd81abf92ceee0c4d80c39 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_17/hellaswag_2025-12-04T22-57-38.905709.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.4971121290579566, + "acc_stderr,none": 0.004989698183207822, + "acc_norm,none": 0.7019518024297948, + "acc_norm_stderr,none": 0.004564659775075878 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764857528.625406, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1158411.282771649, + "end_time": 1161173.064076311, + "total_evaluation_time_seconds": "2761.781304661883" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_17/piqa_2025-12-04T22-11-09.472512.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_17/piqa_2025-12-04T22-11-09.472512.json new file mode 100644 index 0000000000000000000000000000000000000000..e91f52a6ebc630d80f7b8706d77c3f0226ff1afa --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_17/piqa_2025-12-04T22-11-09.472512.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7519042437431991, + "acc_stderr,none": 0.010077118315574715, + "acc_norm,none": 0.7622415669205659, + "acc_norm_stderr,none": 0.009932525779525489 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764857416.4184122, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1158299.221973528, + "end_time": 1158383.630745199, + "total_evaluation_time_seconds": "84.40877167112194" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_17/truthfulqa_mc1_2025-12-04T22-09-16.981978.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_17/truthfulqa_mc1_2025-12-04T22-09-16.981978.json new file mode 100644 index 0000000000000000000000000000000000000000..6477a54b64a071339cab888ea72e3a6cdf1f6744 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_17/truthfulqa_mc1_2025-12-04T22-09-16.981978.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.3463892288861689, + "acc_stderr,none": 0.016656997109125136 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764857255.2364745, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1158137.299956328, + "end_time": 1158271.139983706, + "total_evaluation_time_seconds": "133.84002737794071" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_17/winogrande_2025-12-04T21-55-14.642540.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_17/winogrande_2025-12-04T21-55-14.642540.json new file mode 100644 index 0000000000000000000000000000000000000000..6e319c38051d259db27ee7e579a82973861133d7 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_17/winogrande_2025-12-04T21-55-14.642540.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.5919494869771112, + "acc_stderr,none": 0.013812822643745027 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764856450.9681897, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1157332.263267854, + "end_time": 1157428.799582748, + "total_evaluation_time_seconds": "96.53631489397958" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_18/arc_challenge_2025-12-04T23-16-34.795873.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_18/arc_challenge_2025-12-04T23-16-34.795873.json new file mode 100644 index 0000000000000000000000000000000000000000..33db89f09b7938f4d41068a69267f04649125a55 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_18/arc_challenge_2025-12-04T23-16-34.795873.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.48208191126279865, + "acc_stderr,none": 0.01460200558549098, + "acc_norm,none": 0.5307167235494881, + "acc_norm_stderr,none": 0.014583792546304038 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764860963.4720426, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1161845.704458633, + "end_time": 1162308.954042636, + "total_evaluation_time_seconds": "463.2495840031188" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_18/boolq_2025-12-04T23-08-21.126789.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_18/boolq_2025-12-04T23-08-21.126789.json new file mode 100644 index 0000000000000000000000000000000000000000..95bc21f7c362002810263466ce7052f92dac4dad --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_18/boolq_2025-12-04T23-08-21.126789.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7376146788990826, + "acc_stderr,none": 0.007694441156340015 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764860763.6512594, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1161644.166884825, + "end_time": 1161815.285148329, + "total_evaluation_time_seconds": "171.11826350400224" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_18/hellaswag_2025-12-05T00-07-25.693349.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_18/hellaswag_2025-12-05T00-07-25.693349.json new file mode 100644 index 0000000000000000000000000000000000000000..bb03610b86b75179fce3e7b8e563c04412d7a9c6 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_18/hellaswag_2025-12-05T00-07-25.693349.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.4902409878510257, + "acc_stderr,none": 0.004988830884131624, + "acc_norm,none": 0.6937860983867755, + "acc_norm_stderr,none": 0.004599776866717505 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764861718.9868517, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1162602.097616959, + "end_time": 1165359.851690367, + "total_evaluation_time_seconds": "2757.7540734081995" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_18/piqa_2025-12-04T23-21-01.242579.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_18/piqa_2025-12-04T23-21-01.242579.json new file mode 100644 index 0000000000000000000000000000000000000000..a4284e351c7adc2410439d7de05afb26e756795a --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_18/piqa_2025-12-04T23-21-01.242579.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7578890097932536, + "acc_stderr,none": 0.009994371269104366, + "acc_norm,none": 0.76550598476605, + "acc_norm_stderr,none": 0.009885203143240547 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764861613.834577, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1162496.426563665, + "end_time": 1162575.400917032, + "total_evaluation_time_seconds": "78.9743533669971" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_18/truthfulqa_mc1_2025-12-04T23-19-13.794003.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_18/truthfulqa_mc1_2025-12-04T23-19-13.794003.json new file mode 100644 index 0000000000000000000000000000000000000000..a38091600a737f4afa9862106604a998f285089b --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_18/truthfulqa_mc1_2025-12-04T23-19-13.794003.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.29253365973072215, + "acc_stderr,none": 0.015925597445286165 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764861455.1302931, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1162337.347889157, + "end_time": 1162467.952341715, + "total_evaluation_time_seconds": "130.60445255809464" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_18/winogrande_2025-12-04T23-04-57.960749.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_18/winogrande_2025-12-04T23-04-57.960749.json new file mode 100644 index 0000000000000000000000000000000000000000..425424adb3b2244f909b4ece82853d1c1e421924 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_18/winogrande_2025-12-04T23-04-57.960749.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.5832675611681136, + "acc_stderr,none": 0.013856250072796316 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764860622.459022, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1161505.026150834, + "end_time": 1161612.119124601, + "total_evaluation_time_seconds": "107.09297376684844" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_19/arc_challenge_2025-12-05T00-25-48.231072.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_19/arc_challenge_2025-12-05T00-25-48.231072.json new file mode 100644 index 0000000000000000000000000000000000000000..068e804a79fcb43138c5aa02f111147aae440b07 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_19/arc_challenge_2025-12-05T00-25-48.231072.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.5059726962457338, + "acc_stderr,none": 0.014610348300255793, + "acc_norm,none": 0.5580204778156996, + "acc_norm_stderr,none": 0.014512682523128345 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764865119.1958323, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1166000.67058062, + "end_time": 1166462.389424037, + "total_evaluation_time_seconds": "461.71884341700934" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_19/boolq_2025-12-05T00-17-38.759034.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_19/boolq_2025-12-05T00-17-38.759034.json new file mode 100644 index 0000000000000000000000000000000000000000..f2af69b178b29c121aed56bc91efb8039e8c4043 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_19/boolq_2025-12-05T00-17-38.759034.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7452599388379205, + "acc_stderr,none": 0.007620703281690058 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764864926.2872143, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1165808.967353038, + "end_time": 1165972.91737, + "total_evaluation_time_seconds": "163.95001696189865" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_19/hellaswag_2025-12-05T01-16-34.278139.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_19/hellaswag_2025-12-05T01-16-34.278139.json new file mode 100644 index 0000000000000000000000000000000000000000..42ecdc7148b795b927765faa7578a3e9fc7c9ae2 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_19/hellaswag_2025-12-05T01-16-34.278139.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.4927305317665804, + "acc_stderr,none": 0.00498925401189576, + "acc_norm,none": 0.7080262895837482, + "acc_norm_stderr,none": 0.004537410615572906 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764865871.9577472, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1166754.687350674, + "end_time": 1169508.436203825, + "total_evaluation_time_seconds": "2753.748853150988" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_19/piqa_2025-12-05T00-30-12.514641.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_19/piqa_2025-12-05T00-30-12.514641.json new file mode 100644 index 0000000000000000000000000000000000000000..72632d207024920f540f7e3034999e0439c55b06 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_19/piqa_2025-12-05T00-30-12.514641.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.749183895538629, + "acc_stderr,none": 0.010113869547069044, + "acc_norm,none": 0.7622415669205659, + "acc_norm_stderr,none": 0.009932525779525489 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764865761.9074314, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1166643.661545287, + "end_time": 1166726.672928653, + "total_evaluation_time_seconds": "83.01138336607255" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_19/truthfulqa_mc1_2025-12-05T00-28-21.987686.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_19/truthfulqa_mc1_2025-12-05T00-28-21.987686.json new file mode 100644 index 0000000000000000000000000000000000000000..3b9c246ac67a8c869865cf4a247c84b42b3f0da7 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_19/truthfulqa_mc1_2025-12-05T00-28-21.987686.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.33659730722154224, + "acc_stderr,none": 0.016542412809494884 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764865607.6509414, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1166490.088293819, + "end_time": 1166616.145761579, + "total_evaluation_time_seconds": "126.05746776005253" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_19/winogrande_2025-12-05T00-14-27.546391.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_19/winogrande_2025-12-05T00-14-27.546391.json new file mode 100644 index 0000000000000000000000000000000000000000..709c254c899ee6c1e76d517151cff2112d231794 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_19/winogrande_2025-12-05T00-14-27.546391.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.5777426992896606, + "acc_stderr,none": 0.01388158203065855 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764864807.9076524, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1165690.729449255, + "end_time": 1165781.703932133, + "total_evaluation_time_seconds": "90.9744828781113" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_2/arc_challenge_2025-12-04T04-47-23.178703.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_2/arc_challenge_2025-12-04T04-47-23.178703.json new file mode 100644 index 0000000000000000000000000000000000000000..63b0aa2992b785b66f25938144bf98f83c4fc68c --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_2/arc_challenge_2025-12-04T04-47-23.178703.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.4684300341296928, + "acc_stderr,none": 0.014582236460866985, + "acc_norm,none": 0.5110921501706485, + "acc_norm_stderr,none": 0.01460779491401305 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764794421.803442, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1095303.997566149, + "end_time": 1095757.3370177, + "total_evaluation_time_seconds": "453.3394515509717" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_2/boolq_2025-12-04T04-39-22.098747.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_2/boolq_2025-12-04T04-39-22.098747.json new file mode 100644 index 0000000000000000000000000000000000000000..e3bcfe649fc34efa88008332da066f6894d3e1bc --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_2/boolq_2025-12-04T04-39-22.098747.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7214067278287462, + "acc_stderr,none": 0.007840936667038511 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764794237.4790864, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1095119.053706014, + "end_time": 1095276.257039034, + "total_evaluation_time_seconds": "157.20333301997744" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_2/hellaswag_2025-12-04T05-37-57.894658.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_2/hellaswag_2025-12-04T05-37-57.894658.json new file mode 100644 index 0000000000000000000000000000000000000000..70851c43d2e6936e5188e5b8089b51fd7b2a843f --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_2/hellaswag_2025-12-04T05-37-57.894658.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.4758016331408086, + "acc_stderr,none": 0.0049839343432504565, + "acc_norm,none": 0.6771559450308704, + "acc_norm_stderr,none": 0.004666080865179635 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764795151.9677494, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1096035.128101232, + "end_time": 1098792.052928447, + "total_evaluation_time_seconds": "2756.924827215029" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_2/piqa_2025-12-04T04-51-33.731729.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_2/piqa_2025-12-04T04-51-33.731729.json new file mode 100644 index 0000000000000000000000000000000000000000..90590a80dc5fe86ad8f1d9b8ab7b7b40a8590430 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_2/piqa_2025-12-04T04-51-33.731729.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7241566920565833, + "acc_stderr,none": 0.010427805502729112, + "acc_norm,none": 0.7252448313384113, + "acc_norm_stderr,none": 0.010415033676676042 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764795050.9359176, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1095933.738647447, + "end_time": 1096007.889988184, + "total_evaluation_time_seconds": "74.15134073700756" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_2/truthfulqa_mc1_2025-12-04T04-49-51.519859.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_2/truthfulqa_mc1_2025-12-04T04-49-51.519859.json new file mode 100644 index 0000000000000000000000000000000000000000..a2aae6072f5d1198d03c0d24cc2f7f66f702cd42 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_2/truthfulqa_mc1_2025-12-04T04-49-51.519859.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.2533659730722154, + "acc_stderr,none": 0.015225899340826842 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764794902.7197876, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1095785.298935171, + "end_time": 1095905.678192545, + "total_evaluation_time_seconds": "120.37925737397745" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_2/winogrande_2025-12-04T04-36-17.075254.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_2/winogrande_2025-12-04T04-36-17.075254.json new file mode 100644 index 0000000000000000000000000000000000000000..235712249855c44532b5164a274cd7397b291604 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_2/winogrande_2025-12-04T04-36-17.075254.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.6471981057616417, + "acc_stderr,none": 0.013429728101788947 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764794121.1839666, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1095003.342376903, + "end_time": 1095091.232603913, + "total_evaluation_time_seconds": "87.89022701000795" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_20/boolq_2025-12-05T01-26-50.197248.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_20/boolq_2025-12-05T01-26-50.197248.json new file mode 100644 index 0000000000000000000000000000000000000000..a19279152e0a2f06bc84b7647b2b303f36da5dad --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_20/boolq_2025-12-05T01-26-50.197248.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7415902140672783, + "acc_stderr,none": 0.007656477542140307 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764869081.721442, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1169962.849344021, + "end_time": 1170124.355615067, + "total_evaluation_time_seconds": "161.50627104612067" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_20/hellaswag_2025-12-05T02-25-47.480254.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_20/hellaswag_2025-12-05T02-25-47.480254.json new file mode 100644 index 0000000000000000000000000000000000000000..dc953789082dfc53218c7362ef8767a7bc8d99f2 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_20/hellaswag_2025-12-05T02-25-47.480254.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.4821748655646286, + "acc_stderr,none": 0.0049866095427490145, + "acc_norm,none": 0.6932881896036646, + "acc_norm_stderr,none": 0.004601862807240239 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764870019.8576872, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1170902.62232011, + "end_time": 1173661.63833751, + "total_evaluation_time_seconds": "2759.016017399961" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_20/piqa_2025-12-05T01-39-21.176091.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_20/piqa_2025-12-05T01-39-21.176091.json new file mode 100644 index 0000000000000000000000000000000000000000..7134f679f3705b1410b82ae426b92a7514e3e2dc --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_20/piqa_2025-12-05T01-39-21.176091.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7470076169749728, + "acc_stderr,none": 0.010142888698862451, + "acc_norm,none": 0.7606093579978237, + "acc_norm_stderr,none": 0.009955884250291713 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764869909.9648423, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1170788.316934241, + "end_time": 1170875.33442684, + "total_evaluation_time_seconds": "87.01749259908684" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_20/truthfulqa_mc1_2025-12-05T01-37-24.923798.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_20/truthfulqa_mc1_2025-12-05T01-37-24.923798.json new file mode 100644 index 0000000000000000000000000000000000000000..a86c4990ea3efeb88e1bbd2f6222a7b38e67fbdb --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_20/truthfulqa_mc1_2025-12-05T01-37-24.923798.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.31334149326805383, + "acc_stderr,none": 0.01623806506905959 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764869750.15302, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1170632.822601214, + "end_time": 1170759.081916803, + "total_evaluation_time_seconds": "126.25931558897719" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_20/winogrande_2025-12-05T01-23-39.827371.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_20/winogrande_2025-12-05T01-23-39.827371.json new file mode 100644 index 0000000000000000000000000000000000000000..7a99afffdca68e093aa2f57049d27b9221aa4cda --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_20/winogrande_2025-12-05T01-23-39.827371.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.5990528808208366, + "acc_stderr,none": 0.013773974554948021 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764868957.3098435, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1169838.321356608, + "end_time": 1169933.985073257, + "total_evaluation_time_seconds": "95.66371664893813" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_21/arc_challenge_2025-12-05T02-44-25.433160.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_21/arc_challenge_2025-12-05T02-44-25.433160.json new file mode 100644 index 0000000000000000000000000000000000000000..c5025e81fee759a778c4438915fa047b08b08ed6 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_21/arc_challenge_2025-12-05T02-44-25.433160.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.47525597269624575, + "acc_stderr,none": 0.014593487694937738, + "acc_norm,none": 0.5085324232081911, + "acc_norm_stderr,none": 0.014609263165632182 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764873434.9323494, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1174317.896744673, + "end_time": 1174779.591348608, + "total_evaluation_time_seconds": "461.6946039351169" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_21/boolq_2025-12-05T02-36-16.741058.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_21/boolq_2025-12-05T02-36-16.741058.json new file mode 100644 index 0000000000000000000000000000000000000000..353dc237da91fee506e68eac35f440d6b9fc1f26 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_21/boolq_2025-12-05T02-36-16.741058.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7397553516819572, + "acc_stderr,none": 0.00767410083315232 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764873253.9067461, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1174137.250043106, + "end_time": 1174290.89836624, + "total_evaluation_time_seconds": "153.6483231340535" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_21/hellaswag_2025-12-05T03-34-53.877040.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_21/hellaswag_2025-12-05T03-34-53.877040.json new file mode 100644 index 0000000000000000000000000000000000000000..d20c461aef6dd64f88de9fd1c32a7bcf68577b57 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_21/hellaswag_2025-12-05T03-34-53.877040.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.46773551085441145, + "acc_stderr,none": 0.0049793818767126154, + "acc_norm,none": 0.6799442342162916, + "acc_norm_stderr,none": 0.004655442766599465 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764874170.1849282, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1175054.969892249, + "end_time": 1177808.035356147, + "total_evaluation_time_seconds": "2753.065463898005" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_21/piqa_2025-12-05T02-48-34.124323.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_21/piqa_2025-12-05T02-48-34.124323.json new file mode 100644 index 0000000000000000000000000000000000000000..5b7ceabb06059a9aa31fbdfd26c07349d3ee34c9 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_21/piqa_2025-12-05T02-48-34.124323.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7415669205658324, + "acc_stderr,none": 0.010213971636773336, + "acc_norm,none": 0.7464635473340587, + "acc_norm_stderr,none": 0.010150090834551772 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764874071.2361355, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1174954.832525746, + "end_time": 1175028.282685304, + "total_evaluation_time_seconds": "73.45015955809504" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_21/truthfulqa_mc1_2025-12-05T02-46-53.461895.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_21/truthfulqa_mc1_2025-12-05T02-46-53.461895.json new file mode 100644 index 0000000000000000000000000000000000000000..c8ed417e14a3f6e74f484e789d7e87f5202f410a --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_21/truthfulqa_mc1_2025-12-05T02-46-53.461895.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.3292533659730722, + "acc_stderr,none": 0.01645126444006823 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764873923.3587966, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1174806.640948371, + "end_time": 1174927.620194476, + "total_evaluation_time_seconds": "120.97924610506743" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_22/arc_challenge_2025-12-05T03-52-44.132352.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_22/arc_challenge_2025-12-05T03-52-44.132352.json new file mode 100644 index 0000000000000000000000000000000000000000..4f18fa8f0d6484f957323040063bb699f344463f --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_22/arc_challenge_2025-12-05T03-52-44.132352.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.4590443686006826, + "acc_stderr,none": 0.01456229107360123, + "acc_norm,none": 0.4991467576791809, + "acc_norm_stderr,none": 0.014611369529813265 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764877545.9234865, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1178429.10023916, + "end_time": 1178878.290715749, + "total_evaluation_time_seconds": "449.19047658890486" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_22/boolq_2025-12-05T03-44-47.631143.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_22/boolq_2025-12-05T03-44-47.631143.json new file mode 100644 index 0000000000000000000000000000000000000000..6f8ee9b573e96ffc5d3414de219fe139d04cb4ef --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_22/boolq_2025-12-05T03-44-47.631143.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7339449541284404, + "acc_stderr,none": 0.007728763786791672 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764877366.0813656, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1178249.141967268, + "end_time": 1178401.789439659, + "total_evaluation_time_seconds": "152.64747239113785" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_22/hellaswag_2025-12-05T04-43-05.058331.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_22/hellaswag_2025-12-05T04-43-05.058331.json new file mode 100644 index 0000000000000000000000000000000000000000..bff55cde6316a2842b85218cf234e6c027957a96 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_22/hellaswag_2025-12-05T04-43-05.058331.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.4654451304521012, + "acc_stderr,none": 0.004977851161904406, + "acc_norm,none": 0.6716789484166501, + "acc_norm_stderr,none": 0.0046864258512533405 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764878268.437532, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1179152.17725552, + "end_time": 1181899.216657407, + "total_evaluation_time_seconds": "2747.0394018872175" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_22/piqa_2025-12-05T03-56-50.766578.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_22/piqa_2025-12-05T03-56-50.766578.json new file mode 100644 index 0000000000000000000000000000000000000000..19efb6c2b45b69784e91e1bfdebc05c7eb09881a --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_22/piqa_2025-12-05T03-56-50.766578.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7415669205658324, + "acc_stderr,none": 0.010213971636773327, + "acc_norm,none": 0.7421109902067464, + "acc_norm_stderr,none": 0.010206956662056269 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764878167.215587, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1179051.031513986, + "end_time": 1179124.924910597, + "total_evaluation_time_seconds": "73.89339661109261" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_22/truthfulqa_mc1_2025-12-05T03-55-09.872052.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_22/truthfulqa_mc1_2025-12-05T03-55-09.872052.json new file mode 100644 index 0000000000000000000000000000000000000000..d4835101baff3112a3790fadb964a6a0b3200e10 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_22/truthfulqa_mc1_2025-12-05T03-55-09.872052.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.3378212974296206, + "acc_stderr,none": 0.016557167322516875 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764878019.7023973, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1178903.219074524, + "end_time": 1179024.030378129, + "total_evaluation_time_seconds": "120.81130360509269" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_22/winogrande_2025-12-05T03-41-48.257315.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_22/winogrande_2025-12-05T03-41-48.257315.json new file mode 100644 index 0000000000000000000000000000000000000000..744004b2cea885fa02af442b9e1806641ff26db5 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_22/winogrande_2025-12-05T03-41-48.257315.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.5880031570639306, + "acc_stderr,none": 0.013833112857645933 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764877251.7784686, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1178134.470787437, + "end_time": 1178222.414765346, + "total_evaluation_time_seconds": "87.94397790916264" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_23/arc_challenge_2025-12-05T05-00-52.773024.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_23/arc_challenge_2025-12-05T05-00-52.773024.json new file mode 100644 index 0000000000000000000000000000000000000000..798acd2358c6cd646fbbde5034f915854bd288bb --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_23/arc_challenge_2025-12-05T05-00-52.773024.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.4709897610921502, + "acc_stderr,none": 0.014586776355294326, + "acc_norm,none": 0.492320819112628, + "acc_norm_stderr,none": 0.01460966744089257 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764881634.2315927, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1182517.749408048, + "end_time": 1182966.931371189, + "total_evaluation_time_seconds": "449.18196314084344" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_23/hellaswag_2025-12-05T05-51-23.012013.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_23/hellaswag_2025-12-05T05-51-23.012013.json new file mode 100644 index 0000000000000000000000000000000000000000..f5e7495dce86edd29156dc6f4e21ddf96e1f69dc --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_23/hellaswag_2025-12-05T05-51-23.012013.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.47161919936267677, + "acc_stderr,none": 0.004981736689518754, + "acc_norm,none": 0.6808404700258912, + "acc_norm_stderr,none": 0.0046519828640434946 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764882365.9343722, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1183249.995085894, + "end_time": 1185997.170267028, + "total_evaluation_time_seconds": "2747.1751811341383" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_23/piqa_2025-12-05T05-05-11.194422.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_23/piqa_2025-12-05T05-05-11.194422.json new file mode 100644 index 0000000000000000000000000000000000000000..71c69cbde64fb38390eae9c54c41728beb72d5bd --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_23/piqa_2025-12-05T05-05-11.194422.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.736126224156692, + "acc_stderr,none": 0.010282996367695568, + "acc_norm,none": 0.7546245919477693, + "acc_norm_stderr,none": 0.010039831320422393 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764882269.1038604, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1183152.971204711, + "end_time": 1183225.352726839, + "total_evaluation_time_seconds": "72.38152212812565" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_23/winogrande_2025-12-05T04-49-56.384106.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_23/winogrande_2025-12-05T04-49-56.384106.json new file mode 100644 index 0000000000000000000000000000000000000000..c840a267cdb6de6c36e7b4c24a5158e618877944 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_23/winogrande_2025-12-05T04-49-56.384106.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.5872138910812944, + "acc_stderr,none": 0.013837060648682084 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764881340.897227, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1182225.785381931, + "end_time": 1182310.541286807, + "total_evaluation_time_seconds": "84.75590487592854" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_24/arc_challenge_2025-12-05T06-09-10.084317.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_24/arc_challenge_2025-12-05T06-09-10.084317.json new file mode 100644 index 0000000000000000000000000000000000000000..dcea1abe44936aae90317410b6fb2d0cbb50e58a --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_24/arc_challenge_2025-12-05T06-09-10.084317.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.45051194539249145, + "acc_stderr,none": 0.014539646098471627, + "acc_norm,none": 0.492320819112628, + "acc_norm_stderr,none": 0.014609667440892574 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764885730.7631621, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1186614.147891105, + "end_time": 1187064.242652464, + "total_evaluation_time_seconds": "450.0947613590397" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_24/hellaswag_2025-12-05T06-59-28.072661.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_24/hellaswag_2025-12-05T06-59-28.072661.json new file mode 100644 index 0000000000000000000000000000000000000000..bd2b896b16372541cfba8cc3ab73f42c271c2cd6 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_24/hellaswag_2025-12-05T06-59-28.072661.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.4865564628560048, + "acc_stderr,none": 0.004987977492042151, + "acc_norm,none": 0.6905994821748656, + "acc_norm_stderr,none": 0.004613018101185318 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764886454.8933978, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1187338.729059778, + "end_time": 1190082.230827661, + "total_evaluation_time_seconds": "2743.5017678830773" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_24/piqa_2025-12-05T06-13-17.201347.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_24/piqa_2025-12-05T06-13-17.201347.json new file mode 100644 index 0000000000000000000000000000000000000000..25c12c5610f31a1b8365523c43cd4e5147d16740 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_24/piqa_2025-12-05T06-13-17.201347.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7366702937976061, + "acc_stderr,none": 0.010276185322196764, + "acc_norm,none": 0.7573449401523396, + "acc_norm_stderr,none": 0.010002002569708693 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764886354.31009, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1187235.773423222, + "end_time": 1187311.359700694, + "total_evaluation_time_seconds": "75.58627747208811" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_24/truthfulqa_mc1_2025-12-05T06-11-34.423217.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_24/truthfulqa_mc1_2025-12-05T06-11-34.423217.json new file mode 100644 index 0000000000000000000000000000000000000000..8a50e7ba3e6fdfb4c4e3a31ef0ceb9e30284aaac --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_24/truthfulqa_mc1_2025-12-05T06-11-34.423217.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.3072215422276622, + "acc_stderr,none": 0.01615020132132302 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764886207.0127976, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1187090.600032641, + "end_time": 1187208.581493421, + "total_evaluation_time_seconds": "117.98146077990532" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_25/arc_challenge_2025-12-05T07-17-33.123245.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_25/arc_challenge_2025-12-05T07-17-33.123245.json new file mode 100644 index 0000000000000000000000000000000000000000..8d0efad9a24a4aa9be77a65b81fe59f5875acad2 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_25/arc_challenge_2025-12-05T07-17-33.123245.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.49658703071672355, + "acc_stderr,none": 0.014611050403244077, + "acc_norm,none": 0.5401023890784983, + "acc_norm_stderr,none": 0.014564318856924848 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764889831.3159235, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1190714.161974205, + "end_time": 1191167.281288602, + "total_evaluation_time_seconds": "453.11931439698674" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_25/piqa_2025-12-05T07-21-38.996237.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_25/piqa_2025-12-05T07-21-38.996237.json new file mode 100644 index 0000000000000000000000000000000000000000..4f9c675e7884044791f55888b5c8d597c95bce27 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_25/piqa_2025-12-05T07-21-38.996237.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7421109902067464, + "acc_stderr,none": 0.010206956662056278, + "acc_norm,none": 0.7622415669205659, + "acc_norm_stderr,none": 0.009932525779525489 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764890456.4589384, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1191339.34144979, + "end_time": 1191413.154483075, + "total_evaluation_time_seconds": "73.81303328508511" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_26/arc_challenge_2025-12-05T08-25-57.718476.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_26/arc_challenge_2025-12-05T08-25-57.718476.json new file mode 100644 index 0000000000000000000000000000000000000000..c376e5fd300c11987843e11acce6fee044ba6c2d --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_26/arc_challenge_2025-12-05T08-25-57.718476.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.5187713310580204, + "acc_stderr,none": 0.014601090150633964, + "acc_norm,none": 0.560580204778157, + "acc_norm_stderr,none": 0.014503747823580122 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764893936.509063, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1194818.81268673, + "end_time": 1195271.876833697, + "total_evaluation_time_seconds": "453.0641469671391" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_26/piqa_2025-12-05T08-30-07.855388.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_26/piqa_2025-12-05T08-30-07.855388.json new file mode 100644 index 0000000000000000000000000000000000000000..2b15788d65f183d9f98ac09d66b11ee624ee9983 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/mlp_26/piqa_2025-12-05T08-30-07.855388.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7475516866158868, + "acc_stderr,none": 0.010135665547362359, + "acc_norm,none": 0.7616974972796517, + "acc_norm_stderr,none": 0.009940334245876212 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764894564.612211, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1195446.101209014, + "end_time": 1195522.013642164, + "total_evaluation_time_seconds": "75.91243314999156" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_26/truthfulqa_mc1_2025-12-03T18-26-57.422026.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_26/truthfulqa_mc1_2025-12-03T18-26-57.422026.json new file mode 100644 index 0000000000000000000000000000000000000000..1240e691e059b325379bfc63e9f2b0c14fa428c5 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_26/truthfulqa_mc1_2025-12-03T18-26-57.422026.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.3402692778457772, + "acc_stderr,none": 0.016586304901762557 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764757520.991894, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1058403.523757749, + "end_time": 1058531.580255094, + "total_evaluation_time_seconds": "128.05649734498002" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_27/hellaswag_2025-12-03T20-24-51.516397.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_27/hellaswag_2025-12-03T20-24-51.516397.json new file mode 100644 index 0000000000000000000000000000000000000000..25c39faa91141af2a7e62412e23cd49ce975d007 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_27/hellaswag_2025-12-03T20-24-51.516397.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.39543915554670384, + "acc_stderr,none": 0.004879455474663814, + "acc_norm,none": 0.592212706632145, + "acc_norm_stderr,none": 0.004904189257891249 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764761951.3880348, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1062832.982018325, + "end_time": 1065605.674708882, + "total_evaluation_time_seconds": "2772.6926905571017" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_27/truthfulqa_mc1_2025-12-03T19-36-19.471813.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_27/truthfulqa_mc1_2025-12-03T19-36-19.471813.json new file mode 100644 index 0000000000000000000000000000000000000000..793dfdd03563da48ebffdbbf06802dd1dc4a1731 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_27/truthfulqa_mc1_2025-12-03T19-36-19.471813.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.31456548347613217, + "acc_stderr,none": 0.016255241993179185 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764761680.0295177, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1062561.417146988, + "end_time": 1062693.629013297, + "total_evaluation_time_seconds": "132.211866308935" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_3/hellaswag_2025-12-02T16-49-18.701194.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_3/hellaswag_2025-12-02T16-49-18.701194.json new file mode 100644 index 0000000000000000000000000000000000000000..3d08c165a87dd3cdd7419e8d5b0ac544eb1663c9 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_3/hellaswag_2025-12-02T16-49-18.701194.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.3805018920533758, + "acc_stderr,none": 0.004845180034271622, + "acc_norm,none": 0.5141406094403506, + "acc_norm_stderr,none": 0.004987785530475673 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764662634.0929341, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 963528.669185828, + "end_time": 966272.859303233, + "total_evaluation_time_seconds": "2744.190117405029" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_3/piqa_2025-12-02T16-03-17.216500.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_3/piqa_2025-12-02T16-03-17.216500.json new file mode 100644 index 0000000000000000000000000000000000000000..8e0a671cdb106b70701bbd1e7e3277d58180f854 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_3/piqa_2025-12-02T16-03-17.216500.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7328618063112078, + "acc_stderr,none": 0.010323440492612435, + "acc_norm,none": 0.7431991294885746, + "acc_norm_stderr,none": 0.010192864802278045 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764662544.688433, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 963439.293551032, + "end_time": 963511.374863647, + "total_evaluation_time_seconds": "72.08131261507515" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_3/truthfulqa_mc1_2025-12-02T16-01-47.440408.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_3/truthfulqa_mc1_2025-12-02T16-01-47.440408.json new file mode 100644 index 0000000000000000000000000000000000000000..7b8c0f3b4bee81c8e3d0df4c6475f83a35e57126 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_3/truthfulqa_mc1_2025-12-02T16-01-47.440408.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.22031823745410037, + "acc_stderr,none": 0.014509045171487272 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764662406.861943, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 963301.310283166, + "end_time": 963421.598296815, + "total_evaluation_time_seconds": "120.28801364894025" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_3/winogrande_2025-12-02T15-49-05.789908.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_3/winogrande_2025-12-02T15-49-05.789908.json new file mode 100644 index 0000000000000000000000000000000000000000..85bc34708f21c0c6648fba97a24bfdc66ec799f4 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_3/winogrande_2025-12-02T15-49-05.789908.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.5714285714285714, + "acc_stderr,none": 0.013908353814606702 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764661669.9011743, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 962563.963300663, + "end_time": 962659.948097932, + "total_evaluation_time_seconds": "95.9847972689895" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_4/piqa_2025-12-02T17-10-37.581322.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_4/piqa_2025-12-02T17-10-37.581322.json new file mode 100644 index 0000000000000000000000000000000000000000..845cd07ee05928ffe14266dc4147eb96570a77bd --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_4/piqa_2025-12-02T17-10-37.581322.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7546245919477693, + "acc_stderr,none": 0.01003983132042239, + "acc_norm,none": 0.7633297062023939, + "acc_norm_stderr,none": 0.009916841655042809 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764666588.4925566, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 967472.269758799, + "end_time": 967551.739647683, + "total_evaluation_time_seconds": "79.4698888839921" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_4/truthfulqa_mc1_2025-12-02T17-08-53.941976.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_4/truthfulqa_mc1_2025-12-02T17-08-53.941976.json new file mode 100644 index 0000000000000000000000000000000000000000..1054283adedeecb251731e7e74638c2c1dab173c --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_4/truthfulqa_mc1_2025-12-02T17-08-53.941976.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.29498164014687883, + "acc_stderr,none": 0.01596440096558965 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764666431.297338, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 967313.801196023, + "end_time": 967448.100331461, + "total_evaluation_time_seconds": "134.29913543805014" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_5/hellaswag_2025-12-02T19-06-22.245422.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_5/hellaswag_2025-12-02T19-06-22.245422.json new file mode 100644 index 0000000000000000000000000000000000000000..7b7ebc69429da65c7f89d431796d33e94aa8f8b5 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_5/hellaswag_2025-12-02T19-06-22.245422.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.5016928898625772, + "acc_stderr,none": 0.0049897528111734184, + "acc_norm,none": 0.7051384186417048, + "acc_norm_stderr,none": 0.004550486186019081 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764670852.615832, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 971735.321579023, + "end_time": 974496.4037283, + "total_evaluation_time_seconds": "2761.0821492769755" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_5/piqa_2025-12-02T18-19-53.012398.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_5/piqa_2025-12-02T18-19-53.012398.json new file mode 100644 index 0000000000000000000000000000000000000000..a21445d947db283dab09000a39893c923f345309 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_5/piqa_2025-12-02T18-19-53.012398.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7480957562568009, + "acc_stderr,none": 0.01012842133508868, + "acc_norm,none": 0.7600652883569097, + "acc_norm_stderr,none": 0.009963625892809545 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764670745.271524, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 971627.760800028, + "end_time": 971707.170321759, + "total_evaluation_time_seconds": "79.40952173096593" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_5/truthfulqa_mc1_2025-12-02T18-18-05.878795.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_5/truthfulqa_mc1_2025-12-02T18-18-05.878795.json new file mode 100644 index 0000000000000000000000000000000000000000..3ab27cadf7d847783e4bc1e4b794324c7a562c1c --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_5/truthfulqa_mc1_2025-12-02T18-18-05.878795.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.2864137086903305, + "acc_stderr,none": 0.015826142439502366 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764670589.5693514, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 971471.838665564, + "end_time": 971600.037053891, + "total_evaluation_time_seconds": "128.19838832702953" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_5/winogrande_2025-12-02T18-04-12.249108.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_5/winogrande_2025-12-02T18-04-12.249108.json new file mode 100644 index 0000000000000000000000000000000000000000..5d860d54bcefa1e4d6fb17cd727b7bcbd801d589 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_5/winogrande_2025-12-02T18-04-12.249108.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.6101026045777427, + "acc_stderr,none": 0.013707547317008463 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764669790.3094869, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 970672.846218816, + "end_time": 970766.406401472, + "total_evaluation_time_seconds": "93.56018265604507" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_6/arc_challenge_2025-12-02T19-24-55.137513.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_6/arc_challenge_2025-12-02T19-24-55.137513.json new file mode 100644 index 0000000000000000000000000000000000000000..2db2a22437e806590f190c0d70b53bee48f1e32d --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_6/arc_challenge_2025-12-02T19-24-55.137513.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.5119453924914675, + "acc_stderr,none": 0.014607220340597167, + "acc_norm,none": 0.5597269624573379, + "acc_norm_stderr,none": 0.014506769524804244 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764674267.1707518, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 975150.03175184, + "end_time": 975609.29570507, + "total_evaluation_time_seconds": "459.2639532299945" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_6/boolq_2025-12-02T19-16-48.239348.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_6/boolq_2025-12-02T19-16-48.239348.json new file mode 100644 index 0000000000000000000000000000000000000000..2183300e9ac4b62319fb2eda5ccc0b24b8ddfa09 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_6/boolq_2025-12-02T19-16-48.239348.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7241590214067278, + "acc_stderr,none": 0.007816978272864554 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764674065.111871, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 974948.276730673, + "end_time": 975122.396219611, + "total_evaluation_time_seconds": "174.1194889380131" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_6/hellaswag_2025-12-02T20-15-47.109226.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_6/hellaswag_2025-12-02T20-15-47.109226.json new file mode 100644 index 0000000000000000000000000000000000000000..44f171afd33c3c322301aaf1106e5f98a53ac052 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_6/hellaswag_2025-12-02T20-15-47.109226.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.49512049392551283, + "acc_stderr,none": 0.0049895437965932895, + "acc_norm,none": 0.698864767974507, + "acc_norm_stderr,none": 0.004578137949298185 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764675021.5568402, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 975903.09501736, + "end_time": 978661.267555375, + "total_evaluation_time_seconds": "2758.1725380149437" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_6/piqa_2025-12-02T19-29-20.708455.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_6/piqa_2025-12-02T19-29-20.708455.json new file mode 100644 index 0000000000000000000000000000000000000000..df18f3bc4323c238b6e5534ab3172d076f2a3ace --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_6/piqa_2025-12-02T19-29-20.708455.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7448313384113167, + "acc_stderr,none": 0.01017157159252183, + "acc_norm,none": 0.7578890097932536, + "acc_norm_stderr,none": 0.009994371269104373 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764674913.459555, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 975796.079014041, + "end_time": 975874.866053993, + "total_evaluation_time_seconds": "78.7870399520034" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_6/truthfulqa_mc1_2025-12-02T19-27-34.021442.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_6/truthfulqa_mc1_2025-12-02T19-27-34.021442.json new file mode 100644 index 0000000000000000000000000000000000000000..f4a92096ca416809389486cf1d91be1c38569518 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_6/truthfulqa_mc1_2025-12-02T19-27-34.021442.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.30966952264381886, + "acc_stderr,none": 0.016185744355144912 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764674755.9356947, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 975637.970130107, + "end_time": 975768.179751877, + "total_evaluation_time_seconds": "130.20962176995818" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_6/winogrande_2025-12-02T19-13-28.187510.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_6/winogrande_2025-12-02T19-13-28.187510.json new file mode 100644 index 0000000000000000000000000000000000000000..27d0de817895c8039bd4ea2388ce206ed1097d68 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_6/winogrande_2025-12-02T19-13-28.187510.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.601420678768745, + "acc_stderr,none": 0.01376035717687384 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764673941.8523216, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 974824.57071318, + "end_time": 974922.344540201, + "total_evaluation_time_seconds": "97.77382702101022" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_7/arc_challenge_2025-12-02T20-34-13.739506.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_7/arc_challenge_2025-12-02T20-34-13.739506.json new file mode 100644 index 0000000000000000000000000000000000000000..fde4793bf0fe605f7b0307d825ac8b2bdd3c975a --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_7/arc_challenge_2025-12-02T20-34-13.739506.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.5025597269624573, + "acc_stderr,none": 0.014611199329843788, + "acc_norm,none": 0.552901023890785, + "acc_norm_stderr,none": 0.014529380160526843 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764678425.1106474, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 979308.51959429, + "end_time": 979767.897871062, + "total_evaluation_time_seconds": "459.3782767720986" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_7/boolq_2025-12-02T20-26-07.133765.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_7/boolq_2025-12-02T20-26-07.133765.json new file mode 100644 index 0000000000000000000000000000000000000000..f1d67ae21801f88ad4087d77dc00db6c8d77bf7d --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_7/boolq_2025-12-02T20-26-07.133765.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7324159021406728, + "acc_stderr,none": 0.00774286295340734 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764678233.2750213, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 979115.794308557, + "end_time": 979281.290819871, + "total_evaluation_time_seconds": "165.49651131406426" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_7/hellaswag_2025-12-02T21-25-03.561085.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_7/hellaswag_2025-12-02T21-25-03.561085.json new file mode 100644 index 0000000000000000000000000000000000000000..47c7e2bb51fae4e47d2a198cbaeee0c030d6b9d8 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_7/hellaswag_2025-12-02T21-25-03.561085.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.49900418243377814, + "acc_stderr,none": 0.004989771515176684, + "acc_norm,none": 0.7069308902609042, + "acc_norm_stderr,none": 0.004542396269999239 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764679176.619672, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 980058.441453695, + "end_time": 982817.719425936, + "total_evaluation_time_seconds": "2759.2779722410487" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_7/piqa_2025-12-02T20-38-36.012403.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_7/piqa_2025-12-02T20-38-36.012403.json new file mode 100644 index 0000000000000000000000000000000000000000..bd1cdec1731a62b626475902e46a1bc7a07cb274 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_7/piqa_2025-12-02T20-38-36.012403.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7568008705114254, + "acc_stderr,none": 0.01000961195385894, + "acc_norm,none": 0.7742110990206746, + "acc_norm_stderr,none": 0.009754980670917335 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764679068.4403317, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 979950.984103707, + "end_time": 980030.170627137, + "total_evaluation_time_seconds": "79.186523429933" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_7/truthfulqa_mc1_2025-12-02T20-36-48.569647.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_7/truthfulqa_mc1_2025-12-02T20-36-48.569647.json new file mode 100644 index 0000000000000000000000000000000000000000..a431b022c1b4da2849430ed568ee5c717cd17045 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_7/truthfulqa_mc1_2025-12-02T20-36-48.569647.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.33047735618115054, + "acc_stderr,none": 0.016466769613698307 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764678913.4458528, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 979796.117732098, + "end_time": 979922.727573234, + "total_evaluation_time_seconds": "126.60984113602899" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_7/winogrande_2025-12-02T20-22-54.258029.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_7/winogrande_2025-12-02T20-22-54.258029.json new file mode 100644 index 0000000000000000000000000000000000000000..9877cbc4348d6ca5600d790e60737f6fc9550045 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_7/winogrande_2025-12-02T20-22-54.258029.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.5911602209944752, + "acc_stderr,none": 0.013816954295135695 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764678106.6582642, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 978989.265999172, + "end_time": 979088.416020949, + "total_evaluation_time_seconds": "99.15002177702263" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_8/arc_challenge_2025-12-02T21-43-41.750488.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_8/arc_challenge_2025-12-02T21-43-41.750488.json new file mode 100644 index 0000000000000000000000000000000000000000..e8b5e473ce22aa8dd3ea30cdc6f4943089215527 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_8/arc_challenge_2025-12-02T21-43-41.750488.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.5051194539249146, + "acc_stderr,none": 0.014610624890309157, + "acc_norm,none": 0.5477815699658704, + "acc_norm_stderr,none": 0.014544519880633832 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764682590.1656919, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 983472.35250332, + "end_time": 983935.908837312, + "total_evaluation_time_seconds": "463.55633399204817" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_8/boolq_2025-12-02T21-35-29.106259.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_8/boolq_2025-12-02T21-35-29.106259.json new file mode 100644 index 0000000000000000000000000000000000000000..486d589969613573d840153fef26c935c74334a2 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_8/boolq_2025-12-02T21-35-29.106259.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7614678899082569, + "acc_stderr,none": 0.007454040738031631 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764682397.4995327, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 983279.976353866, + "end_time": 983443.26460006, + "total_evaluation_time_seconds": "163.2882461939007" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_8/hellaswag_2025-12-02T22-34-38.456238.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_8/hellaswag_2025-12-02T22-34-38.456238.json new file mode 100644 index 0000000000000000000000000000000000000000..0484606fb640d08deb9b77d94ff632e73971b83d --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_8/hellaswag_2025-12-02T22-34-38.456238.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.5061740689105756, + "acc_stderr,none": 0.00498940098472223, + "acc_norm,none": 0.7141007767377017, + "acc_norm_stderr,none": 0.0045091819193229035 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764683348.9483247, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 984231.322618964, + "end_time": 986992.614214403, + "total_evaluation_time_seconds": "2761.291595438961" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_8/piqa_2025-12-02T21-48-09.595057.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_8/piqa_2025-12-02T21-48-09.595057.json new file mode 100644 index 0000000000000000000000000000000000000000..04f552a4b7f8fa1c74d72d901c779f5efcb6ff53 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_8/piqa_2025-12-02T21-48-09.595057.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7480957562568009, + "acc_stderr,none": 0.010128421335088681, + "acc_norm,none": 0.7595212187159956, + "acc_norm_stderr,none": 0.009971345364651071 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764683240.718357, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 984122.475551561, + "end_time": 984203.753363646, + "total_evaluation_time_seconds": "81.277812084998" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_8/truthfulqa_mc1_2025-12-02T21-46-20.034302.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_8/truthfulqa_mc1_2025-12-02T21-46-20.034302.json new file mode 100644 index 0000000000000000000000000000000000000000..cb73151048f99b08292eb158b46c278c597a1d71 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_8/truthfulqa_mc1_2025-12-02T21-46-20.034302.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.29498164014687883, + "acc_stderr,none": 0.015964400965589653 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764683082.147313, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 983964.012022466, + "end_time": 984094.192519051, + "total_evaluation_time_seconds": "130.1804965849733" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_8/winogrande_2025-12-02T21-32-17.222638.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_8/winogrande_2025-12-02T21-32-17.222638.json new file mode 100644 index 0000000000000000000000000000000000000000..e368815fd0bdf5e41984be9f58468f9c04f396bf --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_8/winogrande_2025-12-02T21-32-17.222638.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.5974743488555643, + "acc_stderr,none": 0.013782866831703044 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764682261.390387, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 983143.241994647, + "end_time": 983251.380954963, + "total_evaluation_time_seconds": "108.13896031607874" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_9/arc_challenge_2025-12-02T22-53-14.072394.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_9/arc_challenge_2025-12-02T22-53-14.072394.json new file mode 100644 index 0000000000000000000000000000000000000000..c8add86f0e8122d4b7c3f9962953a3e646e36413 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_9/arc_challenge_2025-12-02T22-53-14.072394.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.4735494880546075, + "acc_stderr,none": 0.014590931358120169, + "acc_norm,none": 0.5179180887372014, + "acc_norm_stderr,none": 0.014602005585490978 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764686760.6418133, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 987643.273565415, + "end_time": 988108.230330122, + "total_evaluation_time_seconds": "464.95676470710896" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_9/boolq_2025-12-02T22-45-01.457815.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_9/boolq_2025-12-02T22-45-01.457815.json new file mode 100644 index 0000000000000000000000000000000000000000..628618c06d479fe921469adb3b684aee3f8acde6 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_9/boolq_2025-12-02T22-45-01.457815.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7412844036697248, + "acc_stderr,none": 0.007659426910763703 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764686564.7337413, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 987446.923983617, + "end_time": 987615.614829338, + "total_evaluation_time_seconds": "168.69084572105203" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_9/hellaswag_2025-12-02T23-44-17.675474.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_9/hellaswag_2025-12-02T23-44-17.675474.json new file mode 100644 index 0000000000000000000000000000000000000000..1d30184e5f2ddc76e0581b2f1e5abee6a8dc5a02 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_9/hellaswag_2025-12-02T23-44-17.675474.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.49153555068711413, + "acc_stderr,none": 0.004989066355449558, + "acc_norm,none": 0.6928898625771759, + "acc_norm_stderr,none": 0.004603527017557776 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764687524.809389, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 988406.252109587, + "end_time": 991171.83169734, + "total_evaluation_time_seconds": "2765.5795877530472" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_9/piqa_2025-12-02T22-57-45.661676.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_9/piqa_2025-12-02T22-57-45.661676.json new file mode 100644 index 0000000000000000000000000000000000000000..1b79fc86ff74e00c043d32228d769ecdd98d3fcb --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_9/piqa_2025-12-02T22-57-45.661676.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7415669205658324, + "acc_stderr,none": 0.010213971636773326, + "acc_norm,none": 0.7562568008705114, + "acc_norm_stderr,none": 0.01001719947150061 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764687416.7824962, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 988298.390686216, + "end_time": 988379.820006291, + "total_evaluation_time_seconds": "81.42932007508352" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_9/truthfulqa_mc1_2025-12-02T22-55-56.818567.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_9/truthfulqa_mc1_2025-12-02T22-55-56.818567.json new file mode 100644 index 0000000000000000000000000000000000000000..ea9d5262606139b0e3fa9ed708636342c4243a76 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_9/truthfulqa_mc1_2025-12-02T22-55-56.818567.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.30354957160342716, + "acc_stderr,none": 0.016095884155386854 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764687253.6759691, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 988135.921676952, + "end_time": 988270.976826492, + "total_evaluation_time_seconds": "135.0551495399559" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_9/winogrande_2025-12-02T22-41-44.073977.json b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_9/winogrande_2025-12-02T22-41-44.073977.json new file mode 100644 index 0000000000000000000000000000000000000000..65ce78ac1ed829f876eb2d9130e743c92f3ef0d7 --- /dev/null +++ b/lm-evaluation-harness/results2/Qwen2.5-7B-quantization-layer/self_attn_9/winogrande_2025-12-02T22-41-44.073977.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.5816890292028414, + "acc_stderr,none": 0.013863669961195898 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda:1", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764686435.6419966, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 987317.312316787, + "end_time": 987418.231912285, + "total_evaluation_time_seconds": "100.91959549800958" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_-1/arc_challenge_2025-12-03T14-46-08.070128.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_-1/arc_challenge_2025-12-03T14-46-08.070128.json new file mode 100644 index 0000000000000000000000000000000000000000..5942bea227f219825252b4e0322c0fc511176cb2 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_-1/arc_challenge_2025-12-03T14-46-08.070128.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.4658703071672355, + "acc_stderr,none": 0.014577311315231099, + "acc_norm,none": 0.5068259385665529, + "acc_norm_stderr,none": 0.014610029151379813 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764744206.6432672, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1045085.22025634, + "end_time": 1045282.228242409, + "total_evaluation_time_seconds": "197.00798606895842" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_-1/boolq_2025-12-03T14-41-59.621801.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_-1/boolq_2025-12-03T14-41-59.621801.json new file mode 100644 index 0000000000000000000000000000000000000000..66c0ac5acd0df707ad4e9c689e1ab143a0ab5d50 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_-1/boolq_2025-12-03T14-41-59.621801.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.789296636085627, + "acc_stderr,none": 0.007132608721450028 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764744039.74835, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1044919.683383522, + "end_time": 1045033.780119075, + "total_evaluation_time_seconds": "114.09673555300105" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_-1/hellaswag_2025-12-03T15-06-02.053815.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_-1/hellaswag_2025-12-03T15-06-02.053815.json new file mode 100644 index 0000000000000000000000000000000000000000..1df0562b9e9492b34a2f36c62c775a848d991aaa --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_-1/hellaswag_2025-12-03T15-06-02.053815.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.536247759410476, + "acc_stderr,none": 0.004976651989757631, + "acc_norm,none": 0.743079067914758, + "acc_norm_stderr,none": 0.004360424536145124 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764744727.6161575, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1045607.670313964, + "end_time": 1046476.211760599, + "total_evaluation_time_seconds": "868.5414466350339" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_-1/piqa_2025-12-03T14-50-43.066949.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_-1/piqa_2025-12-03T14-50-43.066949.json new file mode 100644 index 0000000000000000000000000000000000000000..6dd21b6d3888d662a5f386550121a27a15d01aa1 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_-1/piqa_2025-12-03T14-50-43.066949.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7714907508161044, + "acc_stderr,none": 0.009796313511829519, + "acc_norm,none": 0.7818280739934712, + "acc_norm_stderr,none": 0.009636081958374381 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764744598.3960702, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1045479.65877859, + "end_time": 1045557.225259019, + "total_evaluation_time_seconds": "77.56648042891175" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_-1/truthfulqa_mc1_2025-12-03T14-48-37.169120.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_-1/truthfulqa_mc1_2025-12-03T14-48-37.169120.json new file mode 100644 index 0000000000000000000000000000000000000000..1386bc8bdbdeae6e94bf2ec65688764a3af26ce5 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_-1/truthfulqa_mc1_2025-12-03T14-48-37.169120.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.2386780905752754, + "acc_stderr,none": 0.014922629695456416 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764744453.5453746, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1045333.129317458, + "end_time": 1045431.327498664, + "total_evaluation_time_seconds": "98.19818120601121" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_-1/winogrande_2025-12-03T14-39-14.602083.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_-1/winogrande_2025-12-03T14-39-14.602083.json new file mode 100644 index 0000000000000000000000000000000000000000..e3f0cce9dcdd41ab90fa5ca372ababdac052e009 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_-1/winogrande_2025-12-03T14-39-14.602083.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.7000789265982637, + "acc_stderr,none": 0.01287834752663607 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764743902.9976144, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1044780.654044216, + "end_time": 1044868.760484548, + "total_evaluation_time_seconds": "88.1064403320197" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_0/arc_challenge_2025-12-03T15-20-40.806492.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_0/arc_challenge_2025-12-03T15-20-40.806492.json new file mode 100644 index 0000000000000000000000000000000000000000..4afb7a443144e08a56f8e6ad7ea0a7fbdab3ae61 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_0/arc_challenge_2025-12-03T15-20-40.806492.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.19283276450511946, + "acc_stderr,none": 0.011529055465663325, + "acc_norm,none": 0.24573378839590443, + "acc_norm_stderr,none": 0.012581033453730107 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764746275.3286495, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1047156.761269179, + "end_time": 1047354.964838154, + "total_evaluation_time_seconds": "198.2035689749755" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_0/boolq_2025-12-03T15-16-32.703768.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_0/boolq_2025-12-03T15-16-32.703768.json new file mode 100644 index 0000000000000000000000000000000000000000..effdf9413359fca0a8170f035046a02b206f2de0 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_0/boolq_2025-12-03T15-16-32.703768.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.41712538226299695, + "acc_stderr,none": 0.008624092785001307 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764746105.5556011, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1046985.838300415, + "end_time": 1047106.862081806, + "total_evaluation_time_seconds": "121.02378139097709" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_0/hellaswag_2025-12-03T15-40-46.349287.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_0/hellaswag_2025-12-03T15-40-46.349287.json new file mode 100644 index 0000000000000000000000000000000000000000..730d3d6af2e4e79a514c09c48e00f5e178d76176 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_0/hellaswag_2025-12-03T15-40-46.349287.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.2608046205935073, + "acc_stderr,none": 0.004381761941552698, + "acc_norm,none": 0.26638119896434975, + "acc_norm_stderr,none": 0.0044116243741767 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764746804.6424456, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1047684.369062396, + "end_time": 1048560.507555256, + "total_evaluation_time_seconds": "876.1384928600164" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_0/piqa_2025-12-03T15-25-19.396638.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_0/piqa_2025-12-03T15-25-19.396638.json new file mode 100644 index 0000000000000000000000000000000000000000..44b73822fdbc941e88851f410b46d9d508bba32c --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_0/piqa_2025-12-03T15-25-19.396638.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.5500544069640914, + "acc_stderr,none": 0.01160722083798011, + "acc_norm,none": 0.529923830250272, + "acc_norm_stderr,none": 0.011644913435420155 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764746673.9506612, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1047553.913773324, + "end_time": 1047633.55493096, + "total_evaluation_time_seconds": "79.64115763595328" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_0/truthfulqa_mc1_2025-12-03T15-23-09.189354.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_0/truthfulqa_mc1_2025-12-03T15-23-09.189354.json new file mode 100644 index 0000000000000000000000000000000000000000..32f7694eb9342aa41027b1422a1a5bc789018963 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_0/truthfulqa_mc1_2025-12-03T15-23-09.189354.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.26805385556915545, + "acc_stderr,none": 0.01550620472283456 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764746527.1530507, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1047407.027516974, + "end_time": 1047503.347658974, + "total_evaluation_time_seconds": "96.3201419999823" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_0/winogrande_2025-12-03T15-13-40.863132.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_0/winogrande_2025-12-03T15-13-40.863132.json new file mode 100644 index 0000000000000000000000000000000000000000..ec9a127a9088f7ce4b852e3710276fe35ba2a388 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_0/winogrande_2025-12-03T15-13-40.863132.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.4964483030781373, + "acc_stderr,none": 0.014052131146915873 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764745962.150682, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1046843.132337021, + "end_time": 1046935.021479838, + "total_evaluation_time_seconds": "91.88914281700272" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_1/arc_challenge_2025-12-03T15-55-31.430483.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_1/arc_challenge_2025-12-03T15-55-31.430483.json new file mode 100644 index 0000000000000000000000000000000000000000..92aee2e636cd4dc6e0181c85a8f3f55b22e9ba73 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_1/arc_challenge_2025-12-03T15-55-31.430483.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.197098976109215, + "acc_stderr,none": 0.011625047669880638, + "acc_norm,none": 0.2167235494880546, + "acc_norm_stderr,none": 0.012040156713481189 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764748366.2173953, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1049246.256045502, + "end_time": 1049445.588789658, + "total_evaluation_time_seconds": "199.33274415601045" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_1/boolq_2025-12-03T15-51-20.376175.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_1/boolq_2025-12-03T15-51-20.376175.json new file mode 100644 index 0000000000000000000000000000000000000000..472ee9ac34cf97d81af1e69e75b6739a8c812689 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_1/boolq_2025-12-03T15-51-20.376175.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.582262996941896, + "acc_stderr,none": 0.008625883905552705 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764748190.225021, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1049070.425710754, + "end_time": 1049194.534522023, + "total_evaluation_time_seconds": "124.1088112690486" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_1/hellaswag_2025-12-03T16-15-20.938917.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_1/hellaswag_2025-12-03T16-15-20.938917.json new file mode 100644 index 0000000000000000000000000000000000000000..1fdd3bdfc2fbcca00cb121f0a22352292a3eceb4 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_1/hellaswag_2025-12-03T16-15-20.938917.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.277733519219279, + "acc_stderr,none": 0.00446965904282477, + "acc_norm,none": 0.3073093009360685, + "acc_norm_stderr,none": 0.004604357610190312 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764748882.7459636, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1049763.787450867, + "end_time": 1050635.097222776, + "total_evaluation_time_seconds": "871.3097719089128" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_1/piqa_2025-12-03T15-59-58.958842.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_1/piqa_2025-12-03T15-59-58.958842.json new file mode 100644 index 0000000000000000000000000000000000000000..9f3e51d5f84624553528b0422c17527843ac9716 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_1/piqa_2025-12-03T15-59-58.958842.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.6267682263329706, + "acc_stderr,none": 0.011284653078254896, + "acc_norm,none": 0.6240478781284005, + "acc_norm_stderr,none": 0.011301098166895718 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764748753.7311916, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1049635.80563165, + "end_time": 1049713.116836632, + "total_evaluation_time_seconds": "77.31120498199016" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_1/truthfulqa_mc1_2025-12-03T15-57-53.308825.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_1/truthfulqa_mc1_2025-12-03T15-57-53.308825.json new file mode 100644 index 0000000000000000000000000000000000000000..a2a209327558e1681a107e74036bb22f3c917f2f --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_1/truthfulqa_mc1_2025-12-03T15-57-53.308825.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.24479804161566707, + "acc_stderr,none": 0.015051869486715006 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764748616.8959906, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1049496.210836631, + "end_time": 1049587.466908453, + "total_evaluation_time_seconds": "91.2560718220193" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_1/winogrande_2025-12-03T15-48-25.926663.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_1/winogrande_2025-12-03T15-48-25.926663.json new file mode 100644 index 0000000000000000000000000000000000000000..ea23ae15ab04d56655980eebb3a853dfb497e5bf --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_1/winogrande_2025-12-03T15-48-25.926663.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.500394632991318, + "acc_stderr,none": 0.014052481306049516 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764748049.2423604, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1048929.169675965, + "end_time": 1049020.084925357, + "total_evaluation_time_seconds": "90.91524939192459" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_10/arc_challenge_2025-12-03T21-11-23.831286.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_10/arc_challenge_2025-12-03T21-11-23.831286.json new file mode 100644 index 0000000000000000000000000000000000000000..0ade6e3f92d53df8e76353c841bde52c829d01c7 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_10/arc_challenge_2025-12-03T21-11-23.831286.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.4180887372013652, + "acc_stderr,none": 0.014413988396996072, + "acc_norm,none": 0.46928327645051193, + "acc_norm_stderr,none": 0.014583792546304038 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764767315.283227, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1068195.036515771, + "end_time": 1068397.989607272, + "total_evaluation_time_seconds": "202.95309150102548" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_10/boolq_2025-12-03T21-07-08.925675.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_10/boolq_2025-12-03T21-07-08.925675.json new file mode 100644 index 0000000000000000000000000000000000000000..fd921d4c010cad76dedb1bc05e36d3190ba3fdaf --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_10/boolq_2025-12-03T21-07-08.925675.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.744954128440367, + "acc_stderr,none": 0.007623711520701584 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764767133.7705038, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1068013.152773856, + "end_time": 1068143.083956846, + "total_evaluation_time_seconds": "129.93118299008347" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_10/hellaswag_2025-12-03T21-31-38.838710.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_10/hellaswag_2025-12-03T21-31-38.838710.json new file mode 100644 index 0000000000000000000000000000000000000000..0fa9548c65c0086e92c47129a1c01309e6aab516 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_10/hellaswag_2025-12-03T21-31-38.838710.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.510655247958574, + "acc_stderr,none": 0.004988648260010021, + "acc_norm,none": 0.7103166699860586, + "acc_norm_stderr,none": 0.004526883021027625 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764767857.1701002, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1068736.644280726, + "end_time": 1069612.997007864, + "total_evaluation_time_seconds": "876.3527271379717" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_10/piqa_2025-12-03T21-16-11.292828.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_10/piqa_2025-12-03T21-16-11.292828.json new file mode 100644 index 0000000000000000000000000000000000000000..121929c4f39f7a095e64909f83337f4a6870c1c0 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_10/piqa_2025-12-03T21-16-11.292828.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7611534276387377, + "acc_stderr,none": 0.009948120385337494, + "acc_norm,none": 0.764417845484222, + "acc_norm_stderr,none": 0.009901067586473904 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764767720.8706086, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1068602.633657684, + "end_time": 1068685.451138483, + "total_evaluation_time_seconds": "82.81748079881072" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_10/truthfulqa_mc1_2025-12-03T21-13-57.957308.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_10/truthfulqa_mc1_2025-12-03T21-13-57.957308.json new file mode 100644 index 0000000000000000000000000000000000000000..e9aa4931fdccf40dc4c3b52aa196a9847faa15b0 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_10/truthfulqa_mc1_2025-12-03T21-13-57.957308.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.2252141982864137, + "acc_stderr,none": 0.01462324076802348 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764767570.383735, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1068451.253544672, + "end_time": 1068552.115392087, + "total_evaluation_time_seconds": "100.86184741510078" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_10/winogrande_2025-12-03T21-04-07.599197.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_10/winogrande_2025-12-03T21-04-07.599197.json new file mode 100644 index 0000000000000000000000000000000000000000..e1df0cf2c58408a43a2bc409d9c96f23ccddd5bb --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_10/winogrande_2025-12-03T21-04-07.599197.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.67008681925809, + "acc_stderr,none": 0.013214432542517543 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764766982.1104975, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1067862.425858209, + "end_time": 1067961.75752923, + "total_evaluation_time_seconds": "99.33167102118023" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_11/arc_challenge_2025-12-03T21-46-42.349504.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_11/arc_challenge_2025-12-03T21-46-42.349504.json new file mode 100644 index 0000000000000000000000000000000000000000..de53de62a3601f6d590f5444afaeed62d7736fcf --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_11/arc_challenge_2025-12-03T21-46-42.349504.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.4112627986348123, + "acc_stderr,none": 0.014379441068522068, + "acc_norm,none": 0.4598976109215017, + "acc_norm_stderr,none": 0.014564318856924848 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764769429.0052528, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1070310.994099481, + "end_time": 1070516.507670354, + "total_evaluation_time_seconds": "205.51357087306678" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_11/boolq_2025-12-03T21-42-28.051326.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_11/boolq_2025-12-03T21-42-28.051326.json new file mode 100644 index 0000000000000000000000000000000000000000..04831b8660097b45774588fab3485148232ea456 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_11/boolq_2025-12-03T21-42-28.051326.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7513761467889908, + "acc_stderr,none": 0.007559492458080531 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764769260.983399, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1070140.850937021, + "end_time": 1070262.209640554, + "total_evaluation_time_seconds": "121.35870353295468" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_11/hellaswag_2025-12-03T22-07-20.603303.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_11/hellaswag_2025-12-03T22-07-20.603303.json new file mode 100644 index 0000000000000000000000000000000000000000..fe51a54eb64115770ad790078ee85ba513911b2e --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_11/hellaswag_2025-12-03T22-07-20.603303.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.512248556064529, + "acc_stderr,none": 0.00498828398163104, + "acc_norm,none": 0.7090221071499702, + "acc_norm_stderr,none": 0.004532850566893524 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764769994.711206, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1070875.692253033, + "end_time": 1071754.761556316, + "total_evaluation_time_seconds": "879.0693032830022" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_11/piqa_2025-12-03T21-51-50.036650.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_11/piqa_2025-12-03T21-51-50.036650.json new file mode 100644 index 0000000000000000000000000000000000000000..52eb155bfde6575bb9b5c63e085ad6f1bd4dc1eb --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_11/piqa_2025-12-03T21-51-50.036650.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7595212187159956, + "acc_stderr,none": 0.00997134536465107, + "acc_norm,none": 0.7633297062023939, + "acc_norm_stderr,none": 0.009916841655042807 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764769858.0158248, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1070737.789716243, + "end_time": 1070824.19501438, + "total_evaluation_time_seconds": "86.40529813687317" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_11/truthfulqa_mc1_2025-12-03T21-49-32.888042.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_11/truthfulqa_mc1_2025-12-03T21-49-32.888042.json new file mode 100644 index 0000000000000000000000000000000000000000..b1f7c258f7726b36d71f123e6a67df355b6b0c75 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_11/truthfulqa_mc1_2025-12-03T21-49-32.888042.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.2423500611995104, + "acc_stderr,none": 0.015000674373570342 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764769688.4608212, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1070568.675975155, + "end_time": 1070687.046489483, + "total_evaluation_time_seconds": "118.37051432789303" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_11/winogrande_2025-12-03T21-39-33.614057.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_11/winogrande_2025-12-03T21-39-33.614057.json new file mode 100644 index 0000000000000000000000000000000000000000..ec869b38f3f87050e50d54415e344cad65d09c1a --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_11/winogrande_2025-12-03T21-39-33.614057.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.6819258089976322, + "acc_stderr,none": 0.013089285079884678 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764769106.5284665, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1069984.584855946, + "end_time": 1070087.772250126, + "total_evaluation_time_seconds": "103.18739418010227" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_12/arc_challenge_2025-12-03T22-22-43.700171.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_12/arc_challenge_2025-12-03T22-22-43.700171.json new file mode 100644 index 0000000000000000000000000000000000000000..a2ec490ff0f879cf5e703d54866499bed9053b53 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_12/arc_challenge_2025-12-03T22-22-43.700171.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.3993174061433447, + "acc_stderr,none": 0.014312094557946697, + "acc_norm,none": 0.454778156996587, + "acc_norm_stderr,none": 0.014551507060836353 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764771588.9421926, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1072468.738299201, + "end_time": 1072677.858301228, + "total_evaluation_time_seconds": "209.1200020269025" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_12/boolq_2025-12-03T22-18-22.362823.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_12/boolq_2025-12-03T22-18-22.362823.json new file mode 100644 index 0000000000000000000000000000000000000000..e5b378553b45886305d25d4d58bbc4bcdb59c83f --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_12/boolq_2025-12-03T22-18-22.362823.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7556574923547401, + "acc_stderr,none": 0.0075154423789955075 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764771399.4116232, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1072278.548747011, + "end_time": 1072416.520974997, + "total_evaluation_time_seconds": "137.97222798597068" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_12/hellaswag_2025-12-03T22-42-53.029979.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_12/hellaswag_2025-12-03T22-42-53.029979.json new file mode 100644 index 0000000000000000000000000000000000000000..79107f01c41413508802fedf965cc20e07ff40ec --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_12/hellaswag_2025-12-03T22-42-53.029979.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.5153355905198168, + "acc_stderr,none": 0.004987433862274578, + "acc_norm,none": 0.714299940250946, + "acc_norm_stderr,none": 0.004508239594503837 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764772133.5652628, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1073015.938061438, + "end_time": 1073887.188296483, + "total_evaluation_time_seconds": "871.2502350448631" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_12/piqa_2025-12-03T22-27-30.446280.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_12/piqa_2025-12-03T22-27-30.446280.json new file mode 100644 index 0000000000000000000000000000000000000000..8430332b151f8e89f42cc79af9f4a21b3c053b13 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_12/piqa_2025-12-03T22-27-30.446280.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.750272034820457, + "acc_stderr,none": 0.010099232969867492, + "acc_norm,none": 0.7633297062023939, + "acc_norm_stderr,none": 0.009916841655042806 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764772001.5955331, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1072882.682032106, + "end_time": 1072964.604595239, + "total_evaluation_time_seconds": "81.9225631330628" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_12/winogrande_2025-12-03T22-15-12.101928.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_12/winogrande_2025-12-03T22-15-12.101928.json new file mode 100644 index 0000000000000000000000000000000000000000..52bda5b19ae60952ec5e4a098c50ced3667cc644 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_12/winogrande_2025-12-03T22-15-12.101928.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.654301499605367, + "acc_stderr,none": 0.013366596951934383 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764771255.5588005, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1072133.753989476, + "end_time": 1072226.260250186, + "total_evaluation_time_seconds": "92.50626071006991" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_13/arc_challenge_2025-12-03T22-57-52.528568.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_13/arc_challenge_2025-12-03T22-57-52.528568.json new file mode 100644 index 0000000000000000000000000000000000000000..d1f4809bd3f3f05c8ef8c0e6b2cff014d5fcc94c --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_13/arc_challenge_2025-12-03T22-57-52.528568.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.4129692832764505, + "acc_stderr,none": 0.01438834493539833, + "acc_norm,none": 0.45733788395904434, + "acc_norm_stderr,none": 0.014558106543924067 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764773700.929249, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1074581.359338417, + "end_time": 1074786.686617083, + "total_evaluation_time_seconds": "205.32727866596542" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_13/boolq_2025-12-03T22-53-36.481234.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_13/boolq_2025-12-03T22-53-36.481234.json new file mode 100644 index 0000000000000000000000000000000000000000..9d55402b2758503f89753835db4e5cfb36b1010b --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_13/boolq_2025-12-03T22-53-36.481234.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7376146788990826, + "acc_stderr,none": 0.007694441156339991 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764773526.1823382, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1074406.023302062, + "end_time": 1074530.639509795, + "total_evaluation_time_seconds": "124.61620773305185" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_13/hellaswag_2025-12-03T23-18-12.624070.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_13/hellaswag_2025-12-03T23-18-12.624070.json new file mode 100644 index 0000000000000000000000000000000000000000..abdd3c78bcd79133fda1a25d5664b8149667db7d --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_13/hellaswag_2025-12-03T23-18-12.624070.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.5127464648476399, + "acc_stderr,none": 0.00498815974474249, + "acc_norm,none": 0.7141007767377017, + "acc_norm_stderr,none": 0.004509181919322845 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764774244.1394274, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1075124.034638964, + "end_time": 1076006.782332836, + "total_evaluation_time_seconds": "882.7476938718464" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_13/piqa_2025-12-03T23-02-39.061334.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_13/piqa_2025-12-03T23-02-39.061334.json new file mode 100644 index 0000000000000000000000000000000000000000..df5f88fdf9c8962a51039c925bf2b9c16185da92 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_13/piqa_2025-12-03T23-02-39.061334.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7616974972796517, + "acc_stderr,none": 0.009940334245876205, + "acc_norm,none": 0.7682263329706203, + "acc_norm_stderr,none": 0.009845143772794026 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764774108.4993587, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1074988.835275176, + "end_time": 1075073.219667128, + "total_evaluation_time_seconds": "84.3843919520732" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_13/truthfulqa_mc1_2025-12-03T23-00-23.899003.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_13/truthfulqa_mc1_2025-12-03T23-00-23.899003.json new file mode 100644 index 0000000000000000000000000000000000000000..ca64d3cd4e784e13248677172092d4c6a2b898e7 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_13/truthfulqa_mc1_2025-12-03T23-00-23.899003.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.24112607099143207, + "acc_stderr,none": 0.014974827279752334 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764773956.2777689, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1074837.412631329, + "end_time": 1074938.057200739, + "total_evaluation_time_seconds": "100.64456941001117" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_13/winogrande_2025-12-03T22-50-40.931791.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_13/winogrande_2025-12-03T22-50-40.931791.json new file mode 100644 index 0000000000000000000000000000000000000000..41f7ea8cc6578b95bfaf3e9e57a0198bd6eb045f --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_13/winogrande_2025-12-03T22-50-40.931791.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.6898184688239937, + "acc_stderr,none": 0.013000454144859902 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764773378.0687358, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1074258.53973294, + "end_time": 1074355.090146012, + "total_evaluation_time_seconds": "96.55041307187639" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_14/boolq_2025-12-03T23-28-56.537747.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_14/boolq_2025-12-03T23-28-56.537747.json new file mode 100644 index 0000000000000000000000000000000000000000..9ff0cbec005dc0348fd5d8de9ee66b09c2be90e0 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_14/boolq_2025-12-03T23-28-56.537747.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.772782874617737, + "acc_stderr,none": 0.007328950945979895 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764775649.769614, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1076527.914413934, + "end_time": 1076650.696029474, + "total_evaluation_time_seconds": "122.78161553991958" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_14/hellaswag_2025-12-03T23-53-23.414898.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_14/hellaswag_2025-12-03T23-53-23.414898.json new file mode 100644 index 0000000000000000000000000000000000000000..b8d4085cba4a5b9d5258f24a096140611f31d954 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_14/hellaswag_2025-12-03T23-53-23.414898.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.5169288986257717, + "acc_stderr,none": 0.004986920572284416, + "acc_norm,none": 0.7235610436168094, + "acc_norm_stderr,none": 0.004463224445470978 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764776367.8979316, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1077246.409047926, + "end_time": 1078117.573238883, + "total_evaluation_time_seconds": "871.164190956857" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_14/piqa_2025-12-03T23-37-58.722096.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_14/piqa_2025-12-03T23-37-58.722096.json new file mode 100644 index 0000000000000000000000000000000000000000..53ffc4490e20ec1cc58272b5687ce2751a48e2fd --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_14/piqa_2025-12-03T23-37-58.722096.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7633297062023939, + "acc_stderr,none": 0.009916841655042806, + "acc_norm,none": 0.7709466811751904, + "acc_norm_stderr,none": 0.009804509865175504 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764776230.8113513, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1077112.659630816, + "end_time": 1077192.880444668, + "total_evaluation_time_seconds": "80.22081385203637" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_14/truthfulqa_mc1_2025-12-03T23-35-48.848888.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_14/truthfulqa_mc1_2025-12-03T23-35-48.848888.json new file mode 100644 index 0000000000000000000000000000000000000000..42b8dfe941dd7b6f9726d275d3d6cc00e8d9083a --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_14/truthfulqa_mc1_2025-12-03T23-35-48.848888.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.24112607099143207, + "acc_stderr,none": 0.01497482727975233 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764776076.68933, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1076955.10306914, + "end_time": 1077063.007194525, + "total_evaluation_time_seconds": "107.9041253849864" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_14/winogrande_2025-12-03T23-26-03.448151.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_14/winogrande_2025-12-03T23-26-03.448151.json new file mode 100644 index 0000000000000000000000000000000000000000..dd37aa19735c62076abc552ec49ecf75a0048320 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_14/winogrande_2025-12-03T23-26-03.448151.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.6850828729281768, + "acc_stderr,none": 0.013054277568469233 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764775501.1017995, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1076377.534727246, + "end_time": 1076477.606416823, + "total_evaluation_time_seconds": "100.07168957707472" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_15/boolq_2025-12-04T00-03-52.939373.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_15/boolq_2025-12-04T00-03-52.939373.json new file mode 100644 index 0000000000000000000000000000000000000000..97ecf07eea59472e4deca7294416010895f0fd79 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_15/boolq_2025-12-04T00-03-52.939373.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7798165137614679, + "acc_stderr,none": 0.007247381540554081 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764777748.2868686, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1078627.800039948, + "end_time": 1078747.097721764, + "total_evaluation_time_seconds": "119.29768181592226" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_15/hellaswag_2025-12-04T00-28-03.491058.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_15/hellaswag_2025-12-04T00-28-03.491058.json new file mode 100644 index 0000000000000000000000000000000000000000..d4a5132ddeb15530206a962244b2105cd11f5d19 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_15/hellaswag_2025-12-04T00-28-03.491058.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.5129456283608843, + "acc_stderr,none": 0.004988108663179788, + "acc_norm,none": 0.7189802828121888, + "acc_norm_stderr,none": 0.004485784468576672 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764778447.010364, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1079327.831828388, + "end_time": 1080197.649311373, + "total_evaluation_time_seconds": "869.8174829850905" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_15/piqa_2025-12-04T00-12-40.875023.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_15/piqa_2025-12-04T00-12-40.875023.json new file mode 100644 index 0000000000000000000000000000000000000000..1f49e0a80e7237b1657342f1462ab427fea4dbc0 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_15/piqa_2025-12-04T00-12-40.875023.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7616974972796517, + "acc_stderr,none": 0.009940334245876203, + "acc_norm,none": 0.7595212187159956, + "acc_norm_stderr,none": 0.009971345364651076 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764778314.061464, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1079194.757209335, + "end_time": 1079275.033348769, + "total_evaluation_time_seconds": "80.27613943396136" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_15/truthfulqa_mc1_2025-12-04T00-10-30.092339.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_15/truthfulqa_mc1_2025-12-04T00-10-30.092339.json new file mode 100644 index 0000000000000000000000000000000000000000..ba0d42eae963be59bbc2b00d602ec99f27cc9eb1 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_15/truthfulqa_mc1_2025-12-04T00-10-30.092339.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.24357405140758873, + "acc_stderr,none": 0.015026354824910782 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764778165.9423873, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1079045.674619366, + "end_time": 1079144.250285034, + "total_evaluation_time_seconds": "98.57566566811875" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_15/winogrande_2025-12-04T00-01-01.323159.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_15/winogrande_2025-12-04T00-01-01.323159.json new file mode 100644 index 0000000000000000000000000000000000000000..4ec8d28ddc798377913e532a0c88e5518acdd11f --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_15/winogrande_2025-12-04T00-01-01.323159.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.6953433307024467, + "acc_stderr,none": 0.012935646499325318 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764777611.9702878, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1078489.686555964, + "end_time": 1078575.481397279, + "total_evaluation_time_seconds": "85.79484131513163" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_16/arc_challenge_2025-12-04T00-42-46.393974.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_16/arc_challenge_2025-12-04T00-42-46.393974.json new file mode 100644 index 0000000000000000000000000000000000000000..7750af96ce4d8b6506d2302fa5c94ed48214ba19 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_16/arc_challenge_2025-12-04T00-42-46.393974.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.44112627986348124, + "acc_stderr,none": 0.014509747749064661, + "acc_norm,none": 0.4906143344709898, + "acc_norm_stderr,none": 0.014608816322065003 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764779996.9472194, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1080876.338411648, + "end_time": 1081080.55209022, + "total_evaluation_time_seconds": "204.21367857186124" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_16/hellaswag_2025-12-04T01-02-49.433401.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_16/hellaswag_2025-12-04T01-02-49.433401.json new file mode 100644 index 0000000000000000000000000000000000000000..2c90570a91e734ee27311cd47b11958f1eb0bc6e --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_16/hellaswag_2025-12-04T01-02-49.433401.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.5211113324039036, + "acc_stderr,none": 0.004985331652408335, + "acc_norm,none": 0.7284405496912966, + "acc_norm_stderr,none": 0.0044385491525380264 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764780529.9870136, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1081411.093490822, + "end_time": 1082283.591573813, + "total_evaluation_time_seconds": "872.498082990991" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_16/piqa_2025-12-04T00-47-24.619873.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_16/piqa_2025-12-04T00-47-24.619873.json new file mode 100644 index 0000000000000000000000000000000000000000..b7bb0e76a812e8f6caca80629986c7e884ec5693 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_16/piqa_2025-12-04T00-47-24.619873.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7600652883569097, + "acc_stderr,none": 0.009963625892809545, + "acc_norm,none": 0.7714907508161044, + "acc_norm_stderr,none": 0.009796313511829517 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764780399.1571634, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1081279.869730733, + "end_time": 1081358.778205658, + "total_evaluation_time_seconds": "78.90847492497414" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_16/truthfulqa_mc1_2025-12-04T00-45-15.833988.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_16/truthfulqa_mc1_2025-12-04T00-45-15.833988.json new file mode 100644 index 0000000000000000000000000000000000000000..ec3ca5cf8c5ada3e44cfa2542677b58c4cdf9944 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_16/truthfulqa_mc1_2025-12-04T00-45-15.833988.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.24357405140758873, + "acc_stderr,none": 0.01502635482491078 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764780250.0536535, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1081130.787581103, + "end_time": 1081229.992083564, + "total_evaluation_time_seconds": "99.20450246101245" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_16/winogrande_2025-12-04T00-35-43.623096.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_16/winogrande_2025-12-04T00-35-43.623096.json new file mode 100644 index 0000000000000000000000000000000000000000..f903bf771bd0d13522c75e5141fccfcf16329700 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_16/winogrande_2025-12-04T00-35-43.623096.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.6953433307024467, + "acc_stderr,none": 0.012935646499325312 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764779690.0930378, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1080568.977996344, + "end_time": 1080657.781380667, + "total_evaluation_time_seconds": "88.80338432290591" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_17/arc_challenge_2025-12-04T01-17-29.963978.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_17/arc_challenge_2025-12-04T01-17-29.963978.json new file mode 100644 index 0000000000000000000000000000000000000000..9695a03077434d3af3310ebf062849dbc1be33db --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_17/arc_challenge_2025-12-04T01-17-29.963978.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.43430034129692835, + "acc_stderr,none": 0.014484703048857362, + "acc_norm,none": 0.48464163822525597, + "acc_norm_stderr,none": 0.014604496129394916 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764782088.421127, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1082968.544545792, + "end_time": 1083164.122286093, + "total_evaluation_time_seconds": "195.5777403009124" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_17/boolq_2025-12-04T01-13-23.345536.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_17/boolq_2025-12-04T01-13-23.345536.json new file mode 100644 index 0000000000000000000000000000000000000000..7aa3e506720ebdb345adcb660fd77e3cf4157b0d --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_17/boolq_2025-12-04T01-13-23.345536.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.791131498470948, + "acc_stderr,none": 0.00710973401128647 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764781911.4580934, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1082793.114694655, + "end_time": 1082917.503888588, + "total_evaluation_time_seconds": "124.38919393299147" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_17/piqa_2025-12-04T01-22-07.305376.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_17/piqa_2025-12-04T01-22-07.305376.json new file mode 100644 index 0000000000000000000000000000000000000000..fcacf3bf6c7bf4f02344b65cca97e4c7208b3e1d --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_17/piqa_2025-12-04T01-22-07.305376.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7568008705114254, + "acc_stderr,none": 0.010009611953858927, + "acc_norm,none": 0.7693144722524483, + "acc_norm_stderr,none": 0.00982895955098309 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764782476.003178, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1083357.929024492, + "end_time": 1083441.463729548, + "total_evaluation_time_seconds": "83.5347050561104" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_18/arc_challenge_2025-12-04T01-52-22.300182.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_18/arc_challenge_2025-12-04T01-52-22.300182.json new file mode 100644 index 0000000000000000000000000000000000000000..c6e7ff506d322b914577521343943fb82beb7420 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_18/arc_challenge_2025-12-04T01-52-22.300182.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.4445392491467577, + "acc_stderr,none": 0.014521226405627079, + "acc_norm,none": 0.49146757679180886, + "acc_norm_stderr,none": 0.014609263165632182 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764784140.7700648, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1085022.252471589, + "end_time": 1085256.458342033, + "total_evaluation_time_seconds": "234.20587044395506" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_18/boolq_2025-12-04T01-47-38.011224.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_18/boolq_2025-12-04T01-47-38.011224.json new file mode 100644 index 0000000000000000000000000000000000000000..8b3257ef9faadd4b609143cc3495d97210ca5b4b --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_18/boolq_2025-12-04T01-47-38.011224.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7743119266055046, + "acc_stderr,none": 0.007311471957462409 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764783977.889015, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1084859.129041167, + "end_time": 1084972.169532268, + "total_evaluation_time_seconds": "113.04049110109918" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_19/truthfulqa_mc1_2025-12-04T02-28-42.795970.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_19/truthfulqa_mc1_2025-12-04T02-28-42.795970.json new file mode 100644 index 0000000000000000000000000000000000000000..08a3d5acf9669aed44d26992467f691360547eb9 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_19/truthfulqa_mc1_2025-12-04T02-28-42.795970.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.2484700122399021, + "acc_stderr,none": 0.015127427096520695 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764786469.1667137, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1087348.89351792, + "end_time": 1087436.954346763, + "total_evaluation_time_seconds": "88.06082884292118" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_20/boolq_2025-12-04T02-56-27.557271.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_20/boolq_2025-12-04T02-56-27.557271.json new file mode 100644 index 0000000000000000000000000000000000000000..6087f904181ea60ca254d7d1a5af27c2234c1d24 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_20/boolq_2025-12-04T02-56-27.557271.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7902140672782875, + "acc_stderr,none": 0.007121198629128885 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764788099.1393929, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1088980.382826137, + "end_time": 1089101.715486309, + "total_evaluation_time_seconds": "121.33266017213464" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_3/piqa_2025-12-03T17-09-26.987418.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_3/piqa_2025-12-03T17-09-26.987418.json new file mode 100644 index 0000000000000000000000000000000000000000..7d5c43d6f46b39897bfb482767956001c6dbb873 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_3/piqa_2025-12-03T17-09-26.987418.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7366702937976061, + "acc_stderr,none": 0.010276185322196766, + "acc_norm,none": 0.749727965179543, + "acc_norm_stderr,none": 0.010106561880089801 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764752922.596847, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1053802.049658454, + "end_time": 1053881.145782351, + "total_evaluation_time_seconds": "79.09612389700487" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_3/winogrande_2025-12-03T16-57-52.443428.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_3/winogrande_2025-12-03T16-57-52.443428.json new file mode 100644 index 0000000000000000000000000000000000000000..7a10edd10db3f2171911db01962c6797314fdd8d --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_3/winogrande_2025-12-03T16-57-52.443428.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.6527229676400947, + "acc_stderr,none": 0.01338090924975123 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764752215.0443175, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1053093.62039059, + "end_time": 1053186.601473611, + "total_evaluation_time_seconds": "92.98108302103356" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_30/boolq_2025-12-04T08-38-46.880467.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_30/boolq_2025-12-04T08-38-46.880467.json new file mode 100644 index 0000000000000000000000000000000000000000..ea6097748d90a58587376dbeaef11665f7990528 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_30/boolq_2025-12-04T08-38-46.880467.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.6844036697247706, + "acc_stderr,none": 0.008128579858785899 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764808647.9137502, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1109528.17096515, + "end_time": 1109641.038813701, + "total_evaluation_time_seconds": "112.86784855090082" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_31/boolq_2025-12-04T09-13-02.798732.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_31/boolq_2025-12-04T09-13-02.798732.json new file mode 100644 index 0000000000000000000000000000000000000000..707cab8344ec821d8f24d464739ab405b2babe84 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_31/boolq_2025-12-04T09-13-02.798732.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.6553516819571865, + "acc_stderr,none": 0.008312235338398068 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764810703.542673, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1111583.802451596, + "end_time": 1111696.957047313, + "total_evaluation_time_seconds": "113.15459571708925" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_31/hellaswag_2025-12-04T09-36-50.243127.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_31/hellaswag_2025-12-04T09-36-50.243127.json new file mode 100644 index 0000000000000000000000000000000000000000..9f467885516bdc44e10adfcef39a695b3d83750d --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_31/hellaswag_2025-12-04T09-36-50.243127.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.5295757817167894, + "acc_stderr,none": 0.004981044370530808, + "acc_norm,none": 0.7159928301135232, + "acc_norm_stderr,none": 0.004500186424443802 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764811375.2972603, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1112255.848847055, + "end_time": 1113124.401335357, + "total_evaluation_time_seconds": "868.5524883018807" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_31/piqa_2025-12-04T09-21-31.050544.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_31/piqa_2025-12-04T09-21-31.050544.json new file mode 100644 index 0000000000000000000000000000000000000000..fd4dfaecb81f3abfc823cf080f3026a4b28890cb --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_31/piqa_2025-12-04T09-21-31.050544.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7557127312295974, + "acc_stderr,none": 0.010024765172284246, + "acc_norm,none": 0.7568008705114254, + "acc_norm_stderr,none": 0.010009611953858941 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764811248.7656322, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1112129.657208685, + "end_time": 1112205.208862029, + "total_evaluation_time_seconds": "75.55165334395133" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_4/hellaswag_2025-12-03T18-00-01.906620.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_4/hellaswag_2025-12-03T18-00-01.906620.json new file mode 100644 index 0000000000000000000000000000000000000000..8ad6444219963b84d8af473c3f0520e839949b94 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_4/hellaswag_2025-12-03T18-00-01.906620.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.43178649671380204, + "acc_stderr,none": 0.004943127583290517, + "acc_norm,none": 0.5828520215096594, + "acc_norm_stderr,none": 0.004920800313232728 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764755155.2622724, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1056034.40069202, + "end_time": 1056916.064992401, + "total_evaluation_time_seconds": "881.6643003809731" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_4/piqa_2025-12-03T17-44-29.240738.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_4/piqa_2025-12-03T17-44-29.240738.json new file mode 100644 index 0000000000000000000000000000000000000000..a285cb3331ca0abd045e6597ea4a9ba43b9a6b4d --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_4/piqa_2025-12-03T17-44-29.240738.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7334058759521219, + "acc_stderr,none": 0.010316749863541367, + "acc_norm,none": 0.7464635473340587, + "acc_norm_stderr,none": 0.01015009083455178 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764755020.8214374, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1055901.462058678, + "end_time": 1055983.399104881, + "total_evaluation_time_seconds": "81.9370462030638" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_4/truthfulqa_mc1_2025-12-03T17-42-16.326733.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_4/truthfulqa_mc1_2025-12-03T17-42-16.326733.json new file mode 100644 index 0000000000000000000000000000000000000000..76fdad214a03788fc319c02b5ba6c604419a2d78 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_4/truthfulqa_mc1_2025-12-03T17-42-16.326733.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.22766217870257038, + "acc_stderr,none": 0.014679255032111068 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764754872.586047, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1055751.653242658, + "end_time": 1055850.485080309, + "total_evaluation_time_seconds": "98.83183765108697" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_4/winogrande_2025-12-03T17-32-55.927411.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_4/winogrande_2025-12-03T17-32-55.927411.json new file mode 100644 index 0000000000000000000000000000000000000000..6661069ac8ae039b3a8d767fbe5d50a50bd3a409 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_4/winogrande_2025-12-03T17-32-55.927411.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.6440410418310971, + "acc_stderr,none": 0.01345674065627396 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764754320.525172, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1055203.402887409, + "end_time": 1055290.085749831, + "total_evaluation_time_seconds": "86.68286242196336" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_5/piqa_2025-12-03T18-19-12.825225.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_5/piqa_2025-12-03T18-19-12.825225.json new file mode 100644 index 0000000000000000000000000000000000000000..2263b2d84d8da594ad6457e8faac1379a0d2fa45 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_5/piqa_2025-12-03T18-19-12.825225.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7426550598476604, + "acc_stderr,none": 0.010199921064792512, + "acc_norm,none": 0.7540805223068553, + "acc_norm_stderr,none": 0.010047331865625203 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764757099.8461318, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1057980.012573078, + "end_time": 1058066.983502877, + "total_evaluation_time_seconds": "86.97092979890294" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_5/truthfulqa_mc1_2025-12-03T18-16-54.198131.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_5/truthfulqa_mc1_2025-12-03T18-16-54.198131.json new file mode 100644 index 0000000000000000000000000000000000000000..62dbdbdfa17c840e96f22c4e0dc83a91670fb9fc --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_5/truthfulqa_mc1_2025-12-03T18-16-54.198131.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.2252141982864137, + "acc_stderr,none": 0.01462324076802348 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764756954.9014003, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1057835.06399422, + "end_time": 1057928.356496468, + "total_evaluation_time_seconds": "93.29250224796124" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_5/winogrande_2025-12-03T18-07-33.616890.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_5/winogrande_2025-12-03T18-07-33.616890.json new file mode 100644 index 0000000000000000000000000000000000000000..27b19c775fd0d8a4014a54484a6076e968e9c7ef --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_5/winogrande_2025-12-03T18-07-33.616890.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.6424625098658248, + "acc_stderr,none": 0.01347000744392069 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764756403.5265195, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1057283.991082129, + "end_time": 1057367.775262764, + "total_evaluation_time_seconds": "83.78418063488789" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_6/arc_challenge_2025-12-03T18-49-28.569979.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_6/arc_challenge_2025-12-03T18-49-28.569979.json new file mode 100644 index 0000000000000000000000000000000000000000..91404e1581e320dd12b6f9abd846e6fb7a375339 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_6/arc_challenge_2025-12-03T18-49-28.569979.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.3848122866894198, + "acc_stderr,none": 0.014218371065251098, + "acc_norm,none": 0.4402730375426621, + "acc_norm_stderr,none": 0.014506769524804243 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764758803.5058458, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1059682.700552643, + "end_time": 1059882.728191341, + "total_evaluation_time_seconds": "200.02763869799674" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_6/boolq_2025-12-03T18-45-17.165479.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_6/boolq_2025-12-03T18-45-17.165479.json new file mode 100644 index 0000000000000000000000000000000000000000..ca24d7cbe217240eb04e7994febdb2ccb9610699 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_6/boolq_2025-12-03T18-45-17.165479.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.6880733944954128, + "acc_stderr,none": 0.008102818891778095 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764758629.7028546, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1059509.034098418, + "end_time": 1059631.3237771, + "total_evaluation_time_seconds": "122.28967868210748" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_6/hellaswag_2025-12-03T19-09-49.297308.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_6/hellaswag_2025-12-03T19-09-49.297308.json new file mode 100644 index 0000000000000000000000000000000000000000..b0321a62ca8bdc5564f8b4d8abd3489692c34add --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_6/hellaswag_2025-12-03T19-09-49.297308.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.4896434973112926, + "acc_stderr,none": 0.004988710917169321, + "acc_norm,none": 0.6763592909778928, + "acc_norm_stderr,none": 0.004669085411342195 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764759346.063973, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1060227.413294152, + "end_time": 1061103.455664678, + "total_evaluation_time_seconds": "876.0423705261201" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_6/truthfulqa_mc1_2025-12-03T18-51-59.429984.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_6/truthfulqa_mc1_2025-12-03T18-51-59.429984.json new file mode 100644 index 0000000000000000000000000000000000000000..945f5a8bb9fc1511fbfae1df14f2f07675d0e09e --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_6/truthfulqa_mc1_2025-12-03T18-51-59.429984.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.211750305997552, + "acc_stderr,none": 0.014302068353925612 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764759054.2104123, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1059934.468739503, + "end_time": 1060033.588085273, + "total_evaluation_time_seconds": "99.11934576998465" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_6/winogrande_2025-12-03T18-42-23.623349.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_6/winogrande_2025-12-03T18-42-23.623349.json new file mode 100644 index 0000000000000000000000000000000000000000..73dc2f8214a5a8682f3151dfc6a6de5722f2b97e --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_6/winogrande_2025-12-03T18-42-23.623349.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.6692975532754538, + "acc_stderr,none": 0.013222435887002683 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764758483.3271818, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1059363.094617291, + "end_time": 1059457.781677015, + "total_evaluation_time_seconds": "94.68705972400494" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_7/arc_challenge_2025-12-03T19-24-48.745534.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_7/arc_challenge_2025-12-03T19-24-48.745534.json new file mode 100644 index 0000000000000000000000000000000000000000..bd1e3d97d258c66a119af356ec292a930c92e213 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_7/arc_challenge_2025-12-03T19-24-48.745534.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.4180887372013652, + "acc_stderr,none": 0.01441398839699607, + "acc_norm,none": 0.46245733788395904, + "acc_norm_stderr,none": 0.014570144495075581 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764760917.3069959, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1061796.966612337, + "end_time": 1062002.903873006, + "total_evaluation_time_seconds": "205.93726066919044" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_7/boolq_2025-12-03T19-20-31.547370.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_7/boolq_2025-12-03T19-20-31.547370.json new file mode 100644 index 0000000000000000000000000000000000000000..3700d2dbd157e596d703d4e26f6c61b68637f70a --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_7/boolq_2025-12-03T19-20-31.547370.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7394495412844037, + "acc_stderr,none": 0.007677021072511151 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764760738.9546385, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1061618.681039448, + "end_time": 1061745.705734075, + "total_evaluation_time_seconds": "127.0246946271509" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_7/hellaswag_2025-12-03T19-45-04.332262.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_7/hellaswag_2025-12-03T19-45-04.332262.json new file mode 100644 index 0000000000000000000000000000000000000000..65e460998a6728f9c6183fea9bd0cb7cab4569d7 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_7/hellaswag_2025-12-03T19-45-04.332262.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.47540330611431986, + "acc_stderr,none": 0.00498374014521862, + "acc_norm,none": 0.6610237004580761, + "acc_norm_stderr,none": 0.004723943549005956 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764761463.7766407, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1062343.738718176, + "end_time": 1063218.490535384, + "total_evaluation_time_seconds": "874.7518172080163" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_7/piqa_2025-12-03T19-29-35.552291.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_7/piqa_2025-12-03T19-29-35.552291.json new file mode 100644 index 0000000000000000000000000000000000000000..f8782e8c5859e14f7fea3ca9909cdbf54aa04fce --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_7/piqa_2025-12-03T19-29-35.552291.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7453754080522307, + "acc_stderr,none": 0.010164432237060487, + "acc_norm,none": 0.7606093579978237, + "acc_norm_stderr,none": 0.009955884250291706 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764761326.6810637, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1062208.951920146, + "end_time": 1062289.71052076, + "total_evaluation_time_seconds": "80.75860061403364" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_7/truthfulqa_mc1_2025-12-03T19-27-23.582754.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_7/truthfulqa_mc1_2025-12-03T19-27-23.582754.json new file mode 100644 index 0000000000000000000000000000000000000000..0ff01a58723b8665afc557a8c57d5009c441c99a --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_7/truthfulqa_mc1_2025-12-03T19-27-23.582754.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.2141982864137087, + "acc_stderr,none": 0.014362148155690469 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764761174.2664459, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1062054.630129283, + "end_time": 1062157.740831375, + "total_evaluation_time_seconds": "103.11070209182799" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_7/winogrande_2025-12-03T19-17-34.161635.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_7/winogrande_2025-12-03T19-17-34.161635.json new file mode 100644 index 0000000000000000000000000000000000000000..eca9c9216c16f6314670d8d3625206da1dfe5a86 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_7/winogrande_2025-12-03T19-17-34.161635.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.6614048934490924, + "acc_stderr,none": 0.013300169865842423 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764760596.733434, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1061474.469371335, + "end_time": 1061568.31994125, + "total_evaluation_time_seconds": "93.8505699147936" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_8/arc_challenge_2025-12-03T20-00-17.602351.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_8/arc_challenge_2025-12-03T20-00-17.602351.json new file mode 100644 index 0000000000000000000000000000000000000000..b75be64e4a9c33e0ce250abb62428dd824565b86 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_8/arc_challenge_2025-12-03T20-00-17.602351.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.41552901023890787, + "acc_stderr,none": 0.014401366641216376, + "acc_norm,none": 0.46075085324232085, + "acc_norm_stderr,none": 0.014566303676636586 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764763040.5084214, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1063920.285235752, + "end_time": 1064131.760209553, + "total_evaluation_time_seconds": "211.47497380105779" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_8/hellaswag_2025-12-03T20-20-42.567455.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_8/hellaswag_2025-12-03T20-20-42.567455.json new file mode 100644 index 0000000000000000000000000000000000000000..61d0b9f6189d631d0ce093d3c17e28f3980e98d0 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_8/hellaswag_2025-12-03T20-20-42.567455.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.5037841067516431, + "acc_stderr,none": 0.004989638507409938, + "acc_norm,none": 0.7032463652658832, + "acc_norm_stderr,none": 0.004558933822995544 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764763599.495323, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1064480.408569427, + "end_time": 1065356.72580285, + "total_evaluation_time_seconds": "876.3172334230039" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_8/piqa_2025-12-03T20-05-14.353833.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_8/piqa_2025-12-03T20-05-14.353833.json new file mode 100644 index 0000000000000000000000000000000000000000..2c1bf6c17aba9d74e3506c8454d4ee6963752690 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_8/piqa_2025-12-03T20-05-14.353833.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7568008705114254, + "acc_stderr,none": 0.010009611953858924, + "acc_norm,none": 0.7584330794341676, + "acc_norm_stderr,none": 0.009986718001804479 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764763456.3900926, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1064336.977830872, + "end_time": 1064428.512106832, + "total_evaluation_time_seconds": "91.53427595994435" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_8/truthfulqa_mc1_2025-12-03T20-02-50.485013.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_8/truthfulqa_mc1_2025-12-03T20-02-50.485013.json new file mode 100644 index 0000000000000000000000000000000000000000..83dbc56a6e3a0ae1ec69d3f09bab6a59ee7d7f99 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_8/truthfulqa_mc1_2025-12-03T20-02-50.485013.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.22276621787025705, + "acc_stderr,none": 0.01456650696139676 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764763299.1785407, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1064181.00062608, + "end_time": 1064284.642962052, + "total_evaluation_time_seconds": "103.6423359720502" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_8/winogrande_2025-12-03T19-52-54.679928.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_8/winogrande_2025-12-03T19-52-54.679928.json new file mode 100644 index 0000000000000000000000000000000000000000..c12c07dba05f7cd97180dda6de91e30d4291bc83 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_8/winogrande_2025-12-03T19-52-54.679928.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.6764009471191792, + "acc_stderr,none": 0.01314888332092316 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764762713.2168355, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1063590.65580272, + "end_time": 1063688.838284199, + "total_evaluation_time_seconds": "98.18248147889972" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_9/arc_challenge_2025-12-03T20-35-49.811782.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_9/arc_challenge_2025-12-03T20-35-49.811782.json new file mode 100644 index 0000000000000000000000000000000000000000..ce1d8fed7685bdb54357bcbfda06cacbf4472fad --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_9/arc_challenge_2025-12-03T20-35-49.811782.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.41467576791808874, + "acc_stderr,none": 0.01439707056440917, + "acc_norm,none": 0.4539249146757679, + "acc_norm_stderr,none": 0.014549221105171864 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764765183.3856544, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1066063.007778802, + "end_time": 1066263.970120426, + "total_evaluation_time_seconds": "200.96234162407927" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_9/boolq_2025-12-03T20-31-37.036300.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_9/boolq_2025-12-03T20-31-37.036300.json new file mode 100644 index 0000000000000000000000000000000000000000..288fe6374e442297d9d8869b01b2859a23c3ccfd --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_9/boolq_2025-12-03T20-31-37.036300.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7504587155963303, + "acc_stderr,none": 0.007568802024186031 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764764991.9248707, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1065871.027163123, + "end_time": 1066011.194395655, + "total_evaluation_time_seconds": "140.1672325320542" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_9/hellaswag_2025-12-03T20-56-17.483851.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_9/hellaswag_2025-12-03T20-56-17.483851.json new file mode 100644 index 0000000000000000000000000000000000000000..a769ff07cddca3ca2a3514b911d15af0f8e88279 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_9/hellaswag_2025-12-03T20-56-17.483851.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.5071698864767975, + "acc_stderr,none": 0.004989268362968729, + "acc_norm,none": 0.7051384186417048, + "acc_norm_stderr,none": 0.00455048618601908 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764765729.9788687, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1066610.682762066, + "end_time": 1067491.641991672, + "total_evaluation_time_seconds": "880.9592296062037" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_9/piqa_2025-12-03T20-40-44.187375.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_9/piqa_2025-12-03T20-40-44.187375.json new file mode 100644 index 0000000000000000000000000000000000000000..0df1fb1f264944029e5411cd7335628e7ee9de0d --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_9/piqa_2025-12-03T20-40-44.187375.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.749183895538629, + "acc_stderr,none": 0.010113869547069044, + "acc_norm,none": 0.7584330794341676, + "acc_norm_stderr,none": 0.009986718001804487 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764765591.4686089, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1066471.12589066, + "end_time": 1066558.345634189, + "total_evaluation_time_seconds": "87.219743528869" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_9/truthfulqa_mc1_2025-12-03T20-38-25.348864.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_9/truthfulqa_mc1_2025-12-03T20-38-25.348864.json new file mode 100644 index 0000000000000000000000000000000000000000..2855e298f78332770553ca9c7e48e2f26c3dfe97 --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_9/truthfulqa_mc1_2025-12-03T20-38-25.348864.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.22276621787025705, + "acc_stderr,none": 0.014566506961396752 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764765436.242231, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1066316.135707367, + "end_time": 1066419.506910445, + "total_evaluation_time_seconds": "103.37120307795703" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_9/winogrande_2025-12-03T20-28-24.438407.json b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_9/winogrande_2025-12-03T20-28-24.438407.json new file mode 100644 index 0000000000000000000000000000000000000000..ba1342c5c5a70f01fc37ad998f2ac8af7501afde --- /dev/null +++ b/lm-evaluation-harness/results3/Llama-3.1-8B-quantization-layer-mlp/mlp_9/winogrande_2025-12-03T20-28-24.438407.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.6677190213101816, + "acc_stderr,none": 0.013238316554236525 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_num_parameters": 8030261248, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764764847.8472464, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_eos_token": [ + "<|end_of_text|>", + "128001" + ], + "tokenizer_bos_token": [ + "<|begin_of_text|>", + "128000" + ], + "eot_token_id": 128001, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Llama-3.1-8B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Llama-3.1-8B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1065726.892349282, + "end_time": 1065818.596717974, + "total_evaluation_time_seconds": "91.70436869189143" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_-1/arc_challenge_2025-12-04T09-51-10.375890.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_-1/arc_challenge_2025-12-04T09-51-10.375890.json new file mode 100644 index 0000000000000000000000000000000000000000..21ef10c831793d70a8c18a6581e76bc0b42eccc6 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_-1/arc_challenge_2025-12-04T09-51-10.375890.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.5051194539249146, + "acc_stderr,none": 0.01461062489030916, + "acc_norm,none": 0.5511945392491467, + "acc_norm_stderr,none": 0.014534599585097658 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 51 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764812900.9227362, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1113783.026871677, + "end_time": 1113984.534244047, + "total_evaluation_time_seconds": "201.50737236998975" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_-1/boolq_2025-12-04T09-46-58.806791.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_-1/boolq_2025-12-04T09-46-58.806791.json new file mode 100644 index 0000000000000000000000000000000000000000..62e4ee898f4425fa79198534bbaae8ed9e52ce3f --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_-1/boolq_2025-12-04T09-46-58.806791.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7559633027522936, + "acc_stderr,none": 0.0075122574995206415 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764812721.470944, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1113602.026679411, + "end_time": 1113732.965232872, + "total_evaluation_time_seconds": "130.93855346087366" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_-1/hellaswag_2025-12-04T10-10-07.435253.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_-1/hellaswag_2025-12-04T10-10-07.435253.json new file mode 100644 index 0000000000000000000000000000000000000000..48da42ec2803873b2360e9a6e1850e1d0d681004 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_-1/hellaswag_2025-12-04T10-10-07.435253.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.509061939852619, + "acc_stderr,none": 0.004988961834874214, + "acc_norm,none": 0.7165903206532563, + "acc_norm_stderr,none": 0.004497325533959629 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764813421.6398456, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1114301.989621036, + "end_time": 1115121.59327327, + "total_evaluation_time_seconds": "819.6036522339564" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_-1/piqa_2025-12-04T09-55-38.053100.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_-1/piqa_2025-12-04T09-55-38.053100.json new file mode 100644 index 0000000000000000000000000000000000000000..49f3dd81785c47d4682af0770f68449ba1697442 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_-1/piqa_2025-12-04T09-55-38.053100.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7557127312295974, + "acc_stderr,none": 0.010024765172284246, + "acc_norm,none": 0.7752992383025027, + "acc_norm_stderr,none": 0.00973828258654838 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764813295.0747423, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1114175.50160029, + "end_time": 1114252.211367037, + "total_evaluation_time_seconds": "76.7097667469643" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_-1/truthfulqa_mc1_2025-12-04T09-53-31.478561.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_-1/truthfulqa_mc1_2025-12-04T09-53-31.478561.json new file mode 100644 index 0000000000000000000000000000000000000000..6f913e5e41f829dab96ccf556a15c0f4e29afc31 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_-1/truthfulqa_mc1_2025-12-04T09-53-31.478561.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.34516523867809057, + "acc_stderr,none": 0.01664310331927494 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764813154.5296862, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1114034.783216975, + "end_time": 1114125.636806167, + "total_evaluation_time_seconds": "90.85358919203281" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_-1/winogrande_2025-12-04T09-43-57.420265.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_-1/winogrande_2025-12-04T09-43-57.420265.json new file mode 100644 index 0000000000000000000000000000000000000000..49890f18ac0419b356d4a80af157d7e197fc62ad --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_-1/winogrande_2025-12-04T09-43-57.420265.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.6022099447513812, + "acc_stderr,none": 0.013755743513749025 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764812593.6544368, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1113473.772906702, + "end_time": 1113551.578573153, + "total_evaluation_time_seconds": "77.80566645087674" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_0/arc_challenge_2025-12-04T10-24-50.993289.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_0/arc_challenge_2025-12-04T10-24-50.993289.json new file mode 100644 index 0000000000000000000000000000000000000000..2cb9e8b297723a21d468c792994ec603183524a5 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_0/arc_challenge_2025-12-04T10-24-50.993289.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.2098976109215017, + "acc_stderr,none": 0.011900548748047423, + "acc_norm,none": 0.257679180887372, + "acc_norm_stderr,none": 0.012780770562768395 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 51 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764814924.1028464, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1115804.516839269, + "end_time": 1116005.151566673, + "total_evaluation_time_seconds": "200.6347274039872" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_0/boolq_2025-12-04T10-20-39.584230.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_0/boolq_2025-12-04T10-20-39.584230.json new file mode 100644 index 0000000000000000000000000000000000000000..22413832f4bf59430525e179940bda9484ce337c --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_0/boolq_2025-12-04T10-20-39.584230.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.6654434250764526, + "acc_stderr,none": 0.008252449845273192 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764814743.8399413, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1115625.049984269, + "end_time": 1115753.742572507, + "total_evaluation_time_seconds": "128.69258823781274" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_0/hellaswag_2025-12-04T10-43-51.332908.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_0/hellaswag_2025-12-04T10-43-51.332908.json new file mode 100644 index 0000000000000000000000000000000000000000..a4995353b86c8442c2d5c5f7b783e91f99152ae0 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_0/hellaswag_2025-12-04T10-43-51.332908.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.35919139613622786, + "acc_stderr,none": 0.004787829168255663, + "acc_norm,none": 0.4582752439753037, + "acc_norm_stderr,none": 0.004972377085916326 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764815444.4211004, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1116323.730459713, + "end_time": 1117145.490955135, + "total_evaluation_time_seconds": "821.7604954221752" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_0/piqa_2025-12-04T10-29-18.819436.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_0/piqa_2025-12-04T10-29-18.819436.json new file mode 100644 index 0000000000000000000000000000000000000000..a7d0a87d4af7fca330981eb04076f88926b6dd81 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_0/piqa_2025-12-04T10-29-18.819436.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.5892274211099021, + "acc_stderr,none": 0.011478565556775776, + "acc_norm,none": 0.5788900979325353, + "acc_norm_stderr,none": 0.011519701059151495 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764815315.5213697, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1116196.121684827, + "end_time": 1116272.977763083, + "total_evaluation_time_seconds": "76.85607825615443" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_0/truthfulqa_mc1_2025-12-04T10-27-11.143679.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_0/truthfulqa_mc1_2025-12-04T10-27-11.143679.json new file mode 100644 index 0000000000000000000000000000000000000000..ac391116f26281d86159693ebc75ac56f3ecbd7b --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_0/truthfulqa_mc1_2025-12-04T10-27-11.143679.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.2631578947368421, + "acc_stderr,none": 0.015415241740237031 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764815174.039967, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1116055.486538981, + "end_time": 1116145.301905645, + "total_evaluation_time_seconds": "89.81536666397005" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_0/winogrande_2025-12-04T10-17-43.184748.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_0/winogrande_2025-12-04T10-17-43.184748.json new file mode 100644 index 0000000000000000000000000000000000000000..99a6082d42a67fad2efd7e2478210f3e923d1ed4 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_0/winogrande_2025-12-04T10-17-43.184748.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.5019731649565904, + "acc_stderr,none": 0.01405237625922564 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764814593.8611104, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1115471.32080534, + "end_time": 1115577.342906612, + "total_evaluation_time_seconds": "106.02210127189755" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_1/arc_challenge_2025-12-04T10-58-38.421166.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_1/arc_challenge_2025-12-04T10-58-38.421166.json new file mode 100644 index 0000000000000000000000000000000000000000..0030cfa249633e5db4dd803303cb5a889de38d52 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_1/arc_challenge_2025-12-04T10-58-38.421166.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.2158703071672355, + "acc_stderr,none": 0.012022975360030668, + "acc_norm,none": 0.26706484641638223, + "acc_norm_stderr,none": 0.012928933196496356 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 51 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764816934.2444718, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1117813.116112054, + "end_time": 1118032.579189435, + "total_evaluation_time_seconds": "219.4630773810204" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_1/boolq_2025-12-04T10-54-04.584388.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_1/boolq_2025-12-04T10-54-04.584388.json new file mode 100644 index 0000000000000000000000000000000000000000..3e08a36f684a5200e02632457bfd2509d223cfc1 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_1/boolq_2025-12-04T10-54-04.584388.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.3782874617737003, + "acc_stderr,none": 0.008482001133931001 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764816746.7153518, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1117626.907206135, + "end_time": 1117758.742777014, + "total_evaluation_time_seconds": "131.8355708788149" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_1/hellaswag_2025-12-04T11-17-31.249947.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_1/hellaswag_2025-12-04T11-17-31.249947.json new file mode 100644 index 0000000000000000000000000000000000000000..6a8ff55699d566ac9f15704b460d3ad79cbc007b --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_1/hellaswag_2025-12-04T11-17-31.249947.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.2593108942441745, + "acc_stderr,none": 0.004373608212561029, + "acc_norm,none": 0.25941047600079664, + "acc_norm_stderr,none": 0.004374153847826759 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764817472.0428863, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1118352.538491732, + "end_time": 1119165.407933428, + "total_evaluation_time_seconds": "812.869441695977" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_1/piqa_2025-12-04T11-03-07.987108.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_1/piqa_2025-12-04T11-03-07.987108.json new file mode 100644 index 0000000000000000000000000000000000000000..3a3bedd05ef590fe35ae7bf2981e92d9d1bc395b --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_1/piqa_2025-12-04T11-03-07.987108.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.5342763873775843, + "acc_stderr,none": 0.011638380213532444, + "acc_norm,none": 0.5108813928182807, + "acc_norm_stderr,none": 0.011663061261117727 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764817344.247645, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1118224.144598545, + "end_time": 1118302.145441611, + "total_evaluation_time_seconds": "78.00084306602366" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_1/truthfulqa_mc1_2025-12-04T11-00-59.356549.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_1/truthfulqa_mc1_2025-12-04T11-00-59.356549.json new file mode 100644 index 0000000000000000000000000000000000000000..5512bd4b4bdf9fe8cdb6bd39d6b9dcb716eb0766 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_1/truthfulqa_mc1_2025-12-04T11-00-59.356549.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.22888616891064872, + "acc_stderr,none": 0.014706994909055025 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764817202.4659846, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1118083.635514426, + "end_time": 1118173.514805256, + "total_evaluation_time_seconds": "89.8792908298783" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_1/winogrande_2025-12-04T10-51-02.117108.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_1/winogrande_2025-12-04T10-51-02.117108.json new file mode 100644 index 0000000000000000000000000000000000000000..9a21533535fc70265e3c4de8c397166c9656d760 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_1/winogrande_2025-12-04T10-51-02.117108.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.4996053670086819, + "acc_stderr,none": 0.014052481306049516 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764816616.873898, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1117496.356980681, + "end_time": 1117576.275451171, + "total_evaluation_time_seconds": "79.91847049002536" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_10/arc_challenge_2025-12-04T16-05-53.608112.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_10/arc_challenge_2025-12-04T16-05-53.608112.json new file mode 100644 index 0000000000000000000000000000000000000000..8b636fb5cc6fcad4b6c49254325826b363984fd3 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_10/arc_challenge_2025-12-04T16-05-53.608112.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.447098976109215, + "acc_stderr,none": 0.01452938016052684, + "acc_norm,none": 0.4991467576791809, + "acc_norm_stderr,none": 0.014611369529813295 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 51 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764835382.0783684, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1136263.038725845, + "end_time": 1136467.766428725, + "total_evaluation_time_seconds": "204.7277028800454" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_10/boolq_2025-12-04T16-01-38.054014.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_10/boolq_2025-12-04T16-01-38.054014.json new file mode 100644 index 0000000000000000000000000000000000000000..4cb87646e7433149968df7cd926e7a74f66c94d8 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_10/boolq_2025-12-04T16-01-38.054014.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7162079510703364, + "acc_stderr,none": 0.007885191054174685 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764835169.7126412, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1136049.865829769, + "end_time": 1136212.212373921, + "total_evaluation_time_seconds": "162.34654415189289" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_10/hellaswag_2025-12-04T16-25-05.558987.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_10/hellaswag_2025-12-04T16-25-05.558987.json new file mode 100644 index 0000000000000000000000000000000000000000..27d39522787960a025c4e7db54d197bcbf535779 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_10/hellaswag_2025-12-04T16-25-05.558987.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.47649870543716394, + "acc_stderr,none": 0.004984266543053114, + "acc_norm,none": 0.6731726747659829, + "acc_norm_stderr,none": 0.0046809492838553145 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764835916.6238585, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1136794.050296016, + "end_time": 1137619.717286581, + "total_evaluation_time_seconds": "825.6669905649032" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_10/piqa_2025-12-04T16-10-28.860785.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_10/piqa_2025-12-04T16-10-28.860785.json new file mode 100644 index 0000000000000000000000000000000000000000..5693dfdf20ad5f7549de76818fa874a826ad3d06 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_10/piqa_2025-12-04T16-10-28.860785.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7372143634385201, + "acc_stderr,none": 0.010269354068140769, + "acc_norm,none": 0.7415669205658324, + "acc_norm_stderr,none": 0.01021397163677333 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764835779.789227, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1136659.139123993, + "end_time": 1136743.019118841, + "total_evaluation_time_seconds": "83.87999484804459" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_10/truthfulqa_mc1_2025-12-04T16-08-11.851975.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_10/truthfulqa_mc1_2025-12-04T16-08-11.851975.json new file mode 100644 index 0000000000000000000000000000000000000000..38e612ba59cd2ed037bd20afa5056fb382405c9c --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_10/truthfulqa_mc1_2025-12-04T16-08-11.851975.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.2631578947368421, + "acc_stderr,none": 0.015415241740237026 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764835632.8479657, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1136515.964380789, + "end_time": 1136606.010311937, + "total_evaluation_time_seconds": "90.04593114787713" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_10/winogrande_2025-12-04T15-58-04.290056.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_10/winogrande_2025-12-04T15-58-04.290056.json new file mode 100644 index 0000000000000000000000000000000000000000..b2c59981907a8785facb0fa90475001ac80dfa28 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_10/winogrande_2025-12-04T15-58-04.290056.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.5927387529597474, + "acc_stderr,none": 0.013808654122417855 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764835034.1584334, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1135913.374693671, + "end_time": 1135998.448375131, + "total_evaluation_time_seconds": "85.07368145999499" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_11/arc_challenge_2025-12-04T16-40-41.176324.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_11/arc_challenge_2025-12-04T16-40-41.176324.json new file mode 100644 index 0000000000000000000000000000000000000000..9e2f8ac8f8426f95ffe9e363f61e75aca76a4dba --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_11/arc_challenge_2025-12-04T16-40-41.176324.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.4598976109215017, + "acc_stderr,none": 0.014564318856924848, + "acc_norm,none": 0.5179180887372014, + "acc_norm_stderr,none": 0.014602005585490971 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 51 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764837456.9649053, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1138337.694368622, + "end_time": 1138555.334633103, + "total_evaluation_time_seconds": "217.64026448107325" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_11/boolq_2025-12-04T16-36-12.335773.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_11/boolq_2025-12-04T16-36-12.335773.json new file mode 100644 index 0000000000000000000000000000000000000000..0fc34aa2d6824183fb8c0e5ca03a63ba922889c5 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_11/boolq_2025-12-04T16-36-12.335773.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7425076452599388, + "acc_stderr,none": 0.007647600166820912 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764837269.2739453, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1138149.766653681, + "end_time": 1138286.494159873, + "total_evaluation_time_seconds": "136.72750619216822" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_11/hellaswag_2025-12-04T16-59-57.214113.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_11/hellaswag_2025-12-04T16-59-57.214113.json new file mode 100644 index 0000000000000000000000000000000000000000..fa0cd6c3db2cba266e96706cfdc426ac65347cf5 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_11/hellaswag_2025-12-04T16-59-57.214113.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.49133638717386974, + "acc_stderr,none": 0.004989032307320742, + "acc_norm,none": 0.6954789882493527, + "acc_norm_stderr,none": 0.00459263736990577 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764838006.863188, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1138887.67720049, + "end_time": 1139711.372430048, + "total_evaluation_time_seconds": "823.6952295580413" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_11/piqa_2025-12-04T16-45-22.429284.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_11/piqa_2025-12-04T16-45-22.429284.json new file mode 100644 index 0000000000000000000000000000000000000000..eb7dbfef6e835ae4ff0386b8b66d18e1e24b3b38 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_11/piqa_2025-12-04T16-45-22.429284.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7486398258977149, + "acc_stderr,none": 0.01012115601681925, + "acc_norm,none": 0.7676822633297062, + "acc_norm_stderr,none": 0.009853201384168241 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764837878.7217805, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1138758.71029324, + "end_time": 1138836.587729026, + "total_evaluation_time_seconds": "77.87743578618392" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_11/truthfulqa_mc1_2025-12-04T16-43-14.159894.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_11/truthfulqa_mc1_2025-12-04T16-43-14.159894.json new file mode 100644 index 0000000000000000000000000000000000000000..26a52b0debf11e71b5b5715cff816683feb54f90 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_11/truthfulqa_mc1_2025-12-04T16-43-14.159894.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.3292533659730722, + "acc_stderr,none": 0.016451264440068235 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764837724.4780588, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1138605.769359002, + "end_time": 1138708.317939418, + "total_evaluation_time_seconds": "102.54858041601256" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_11/winogrande_2025-12-04T16-33-04.779685.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_11/winogrande_2025-12-04T16-33-04.779685.json new file mode 100644 index 0000000000000000000000000000000000000000..8380d89e8fbc936d6c5241e70fc6bb8c5ac5cea9 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_11/winogrande_2025-12-04T16-33-04.779685.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.56353591160221, + "acc_stderr,none": 0.013938569465677023 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764837100.1649323, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1137979.449335553, + "end_time": 1138098.937975161, + "total_evaluation_time_seconds": "119.48863960802555" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_12/arc_challenge_2025-12-04T17-14-48.518551.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_12/arc_challenge_2025-12-04T17-14-48.518551.json new file mode 100644 index 0000000000000000000000000000000000000000..73c67e5f0756ac0ac60a52ddf65275d2ef60645c --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_12/arc_challenge_2025-12-04T17-14-48.518551.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.46075085324232085, + "acc_stderr,none": 0.014566303676636586, + "acc_norm,none": 0.5059726962457338, + "acc_norm_stderr,none": 0.014610348300255795 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 51 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764839516.6975927, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1140396.125647552, + "end_time": 1140602.676579275, + "total_evaluation_time_seconds": "206.55093172285706" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_12/boolq_2025-12-04T17-10-27.118043.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_12/boolq_2025-12-04T17-10-27.118043.json new file mode 100644 index 0000000000000000000000000000000000000000..e60247c4e8a2ad1717c64b9cf064f16d50ca4198 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_12/boolq_2025-12-04T17-10-27.118043.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.691131498470948, + "acc_stderr,none": 0.008080899275231325 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764839323.2513628, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1140200.621754653, + "end_time": 1140341.276443822, + "total_evaluation_time_seconds": "140.6546891690232" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_12/hellaswag_2025-12-04T17-34-02.276956.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_12/hellaswag_2025-12-04T17-34-02.276956.json new file mode 100644 index 0000000000000000000000000000000000000000..1d3f63fb5c1ed8ff02aa533a284ec2121c989c6a --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_12/hellaswag_2025-12-04T17-34-02.276956.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.49213304122684726, + "acc_stderr,none": 0.00498916374765079, + "acc_norm,none": 0.69398526190002, + "acc_norm_stderr,none": 0.004598940722374069 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764840052.5640588, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1140935.019882824, + "end_time": 1141756.435287955, + "total_evaluation_time_seconds": "821.4154051309451" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_12/piqa_2025-12-04T17-19-29.825289.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_12/piqa_2025-12-04T17-19-29.825289.json new file mode 100644 index 0000000000000000000000000000000000000000..3a1fc6d6b440792cf5f37f0e689615ce1bcf2736 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_12/piqa_2025-12-04T17-19-29.825289.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7410228509249184, + "acc_stderr,none": 0.010220966031405612, + "acc_norm,none": 0.7578890097932536, + "acc_norm_stderr,none": 0.009994371269104367 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764839926.8479714, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1140806.658845813, + "end_time": 1140883.983521132, + "total_evaluation_time_seconds": "77.32467531901784" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_12/truthfulqa_mc1_2025-12-04T17-17-21.018066.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_12/truthfulqa_mc1_2025-12-04T17-17-21.018066.json new file mode 100644 index 0000000000000000000000000000000000000000..c2c40e81e169408b65178aa1667f574ce981615e --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_12/truthfulqa_mc1_2025-12-04T17-17-21.018066.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.31946144430844553, + "acc_stderr,none": 0.016322644182960505 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764839776.8372867, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1140656.660500517, + "end_time": 1140755.176084025, + "total_evaluation_time_seconds": "98.51558350794949" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_12/winogrande_2025-12-04T17-07-14.423663.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_12/winogrande_2025-12-04T17-07-14.423663.json new file mode 100644 index 0000000000000000000000000000000000000000..75e60c45ec76bbe453ff08e7e6c421000c24b230 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_12/winogrande_2025-12-04T17-07-14.423663.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.5982636148382005, + "acc_stderr,none": 0.013778439266649494 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764839184.360895, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1140065.037622034, + "end_time": 1140148.581674298, + "total_evaluation_time_seconds": "83.54405226395465" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_13/arc_challenge_2025-12-04T17-48-34.457672.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_13/arc_challenge_2025-12-04T17-48-34.457672.json new file mode 100644 index 0000000000000000000000000000000000000000..7dd4f1e835e82397b6cc2610d379a4f514fdd950 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_13/arc_challenge_2025-12-04T17-48-34.457672.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.46501706484641636, + "acc_stderr,none": 0.01457558392201967, + "acc_norm,none": 0.5136518771331058, + "acc_norm_stderr,none": 0.014605943429860942 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 51 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764841546.9386344, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1142428.349914598, + "end_time": 1142628.6159115, + "total_evaluation_time_seconds": "200.26599690201692" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_13/boolq_2025-12-04T17-44-23.637181.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_13/boolq_2025-12-04T17-44-23.637181.json new file mode 100644 index 0000000000000000000000000000000000000000..5e22c6ef4080f445484aadcee8f76a9f7f9799d4 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_13/boolq_2025-12-04T17-44-23.637181.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.6932721712538227, + "acc_stderr,none": 0.00806530905177178 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764841358.880423, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1142240.934391355, + "end_time": 1142377.795493526, + "total_evaluation_time_seconds": "136.8611021710094" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_13/hellaswag_2025-12-04T18-08-33.213328.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_13/hellaswag_2025-12-04T18-08-33.213328.json new file mode 100644 index 0000000000000000000000000000000000000000..3ab1ea9395ce1fa748c3f4fd031d53d1c2ae370f --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_13/hellaswag_2025-12-04T18-08-33.213328.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.4876518621788488, + "acc_stderr,none": 0.004988259530472481, + "acc_norm,none": 0.6872137024497113, + "acc_norm_stderr,none": 0.004626805906522241 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764842122.1922076, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1143001.920775474, + "end_time": 1143827.371445147, + "total_evaluation_time_seconds": "825.4506696730386" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_13/piqa_2025-12-04T17-53-55.633954.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_13/piqa_2025-12-04T17-53-55.633954.json new file mode 100644 index 0000000000000000000000000000000000000000..ea6ae45aab8dd741e777a0d7683386594848e96e --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_13/piqa_2025-12-04T17-53-55.633954.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7486398258977149, + "acc_stderr,none": 0.010121156016819252, + "acc_norm,none": 0.7622415669205659, + "acc_norm_stderr,none": 0.009932525779525483 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764841945.4957347, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1142826.40191998, + "end_time": 1142949.792335718, + "total_evaluation_time_seconds": "123.39041573787108" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_13/truthfulqa_mc1_2025-12-04T17-51-01.261664.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_13/truthfulqa_mc1_2025-12-04T17-51-01.261664.json new file mode 100644 index 0000000000000000000000000000000000000000..af9d4ccf4146bad491b00ab0b92a57b85930c282 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_13/truthfulqa_mc1_2025-12-04T17-51-01.261664.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.3047735618115055, + "acc_stderr,none": 0.01611412415688246 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764841800.169263, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1142679.5244413, + "end_time": 1142775.420028922, + "total_evaluation_time_seconds": "95.8955876219552" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_13/winogrande_2025-12-04T17-41-15.088238.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_13/winogrande_2025-12-04T17-41-15.088238.json new file mode 100644 index 0000000000000000000000000000000000000000..01195d8f2b8ea53b19f0e224bec944123badc18a --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_13/winogrande_2025-12-04T17-41-15.088238.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.6006314127861089, + "acc_stderr,none": 0.013764933546717617 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764841225.9939988, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1142105.998020397, + "end_time": 1142189.246637728, + "total_evaluation_time_seconds": "83.24861733103171" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_14/arc_challenge_2025-12-04T18-23-21.638478.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_14/arc_challenge_2025-12-04T18-23-21.638478.json new file mode 100644 index 0000000000000000000000000000000000000000..eb529a192fd2d575fd191bab849b6ccb006d5e5c --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_14/arc_challenge_2025-12-04T18-23-21.638478.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.4854948805460751, + "acc_stderr,none": 0.014605241081370053, + "acc_norm,none": 0.5298634812286689, + "acc_norm_stderr,none": 0.014585305840007098 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 51 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764843630.9265912, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1144510.896629473, + "end_time": 1144715.796836529, + "total_evaluation_time_seconds": "204.90020705596544" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_14/boolq_2025-12-04T18-19-04.976083.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_14/boolq_2025-12-04T18-19-04.976083.json new file mode 100644 index 0000000000000000000000000000000000000000..990d72863697e90136bd99656cabbb55cf89d513 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_14/boolq_2025-12-04T18-19-04.976083.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7204892966360856, + "acc_stderr,none": 0.007848840939247374 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764843440.3943563, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1144319.595074075, + "end_time": 1144459.13452821, + "total_evaluation_time_seconds": "139.53945413487963" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_14/hellaswag_2025-12-04T18-42-36.287511.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_14/hellaswag_2025-12-04T18-42-36.287511.json new file mode 100644 index 0000000000000000000000000000000000000000..900392650a61b149a10c6421b19a5b7045f69c45 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_14/hellaswag_2025-12-04T18-42-36.287511.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.49502091216889066, + "acc_stderr,none": 0.0049895339988203485, + "acc_norm,none": 0.6987651862178849, + "acc_norm_stderr,none": 0.004578568617306771 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764844165.576913, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1145045.309710603, + "end_time": 1145870.445553831, + "total_evaluation_time_seconds": "825.1358432280831" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_14/piqa_2025-12-04T18-28-00.291656.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_14/piqa_2025-12-04T18-28-00.291656.json new file mode 100644 index 0000000000000000000000000000000000000000..a221e10033993ab7dcbe57b001132491d4ffb323 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_14/piqa_2025-12-04T18-28-00.291656.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7529923830250272, + "acc_stderr,none": 0.010062268140772622, + "acc_norm,none": 0.7676822633297062, + "acc_norm_stderr,none": 0.009853201384168241 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764844035.5881295, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1144914.396323988, + "end_time": 1144994.449965769, + "total_evaluation_time_seconds": "80.05364178097807" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_14/truthfulqa_mc1_2025-12-04T18-25-49.129998.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_14/truthfulqa_mc1_2025-12-04T18-25-49.129998.json new file mode 100644 index 0000000000000000000000000000000000000000..7769a9f2f86c3a76cafbc938078a7a23543a8c3d --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_14/truthfulqa_mc1_2025-12-04T18-25-49.129998.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.33047735618115054, + "acc_stderr,none": 0.016466769613698303 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764843889.1407547, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1144767.343492017, + "end_time": 1144863.288367135, + "total_evaluation_time_seconds": "95.94487511808984" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_14/winogrande_2025-12-04T18-15-52.021911.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_14/winogrande_2025-12-04T18-15-52.021911.json new file mode 100644 index 0000000000000000000000000000000000000000..056e7bc59e540baab1f1830ef2ee2f1f35d2ffdb --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_14/winogrande_2025-12-04T18-15-52.021911.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.5966850828729282, + "acc_stderr,none": 0.013787257285896238 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764843301.7684214, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1144181.36146718, + "end_time": 1144266.180282997, + "total_evaluation_time_seconds": "84.81881581689231" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_15/arc_challenge_2025-12-04T18-57-30.107177.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_15/arc_challenge_2025-12-04T18-57-30.107177.json new file mode 100644 index 0000000000000000000000000000000000000000..35abf621cce24d8cdb0887a7eb370fe6b46ca226 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_15/arc_challenge_2025-12-04T18-57-30.107177.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.4872013651877133, + "acc_stderr,none": 0.014606603181012538, + "acc_norm,none": 0.5213310580204779, + "acc_norm_stderr,none": 0.014598087973127108 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 51 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764845677.0557857, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1146557.199603307, + "end_time": 1146764.265522972, + "total_evaluation_time_seconds": "207.06591966492124" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_15/boolq_2025-12-04T18-53-10.324633.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_15/boolq_2025-12-04T18-53-10.324633.json new file mode 100644 index 0000000000000000000000000000000000000000..38cf18d72e7c5af12ca425d503a6bae5c2a9ce30 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_15/boolq_2025-12-04T18-53-10.324633.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7275229357798165, + "acc_stderr,none": 0.007787191593866642 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764845480.9450567, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1146361.04876269, + "end_time": 1146504.483047764, + "total_evaluation_time_seconds": "143.43428507400677" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_15/hellaswag_2025-12-04T19-17-04.124809.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_15/hellaswag_2025-12-04T19-17-04.124809.json new file mode 100644 index 0000000000000000000000000000000000000000..6a3eb41f835b6128f3a3c1a984d4c7106d618cee --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_15/hellaswag_2025-12-04T19-17-04.124809.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.4987054371639116, + "acc_stderr,none": 0.004989764686738845, + "acc_norm,none": 0.6984664409480184, + "acc_norm_stderr,none": 0.004579859084500769 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764846226.2013311, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1147105.257712417, + "end_time": 1147938.283053113, + "total_evaluation_time_seconds": "833.0253406958655" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_15/piqa_2025-12-04T19-02-19.784826.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_15/piqa_2025-12-04T19-02-19.784826.json new file mode 100644 index 0000000000000000000000000000000000000000..f2be23dc2f62d1827db9e644e4dc6f4016a2c5c2 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_15/piqa_2025-12-04T19-02-19.784826.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7453754080522307, + "acc_stderr,none": 0.010164432237060492, + "acc_norm,none": 0.750272034820457, + "acc_norm_stderr,none": 0.010099232969867497 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764846088.8819535, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1146968.1543759, + "end_time": 1147053.943107713, + "total_evaluation_time_seconds": "85.78873181296512" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_15/truthfulqa_mc1_2025-12-04T19-00-03.230419.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_15/truthfulqa_mc1_2025-12-04T19-00-03.230419.json new file mode 100644 index 0000000000000000000000000000000000000000..1807415c8159cd4e1417c7022f993289d2e0a13a --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_15/truthfulqa_mc1_2025-12-04T19-00-03.230419.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.2913096695226438, + "acc_stderr,none": 0.01590598704818483 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764845933.9831147, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1146814.774262453, + "end_time": 1146917.38851708, + "total_evaluation_time_seconds": "102.61425462714396" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_15/winogrande_2025-12-04T18-49-55.056732.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_15/winogrande_2025-12-04T18-49-55.056732.json new file mode 100644 index 0000000000000000000000000000000000000000..f1601b7d34e8f8dd2bf82b9467c0c20ac0252d40 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_15/winogrande_2025-12-04T18-49-55.056732.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.5808997632202052, + "acc_stderr,none": 0.013867325192210117 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764845343.2863233, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1146222.395514157, + "end_time": 1146309.21492883, + "total_evaluation_time_seconds": "86.81941467314027" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_16/arc_challenge_2025-12-04T19-32-48.448135.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_16/arc_challenge_2025-12-04T19-32-48.448135.json new file mode 100644 index 0000000000000000000000000000000000000000..9dcc952de049c3ab84870fe9dd13daa68057d619 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_16/arc_challenge_2025-12-04T19-32-48.448135.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.4931740614334471, + "acc_stderr,none": 0.014610029151379813, + "acc_norm,none": 0.5392491467576792, + "acc_norm_stderr,none": 0.014566303676636581 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 51 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764847781.887803, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1148664.030661838, + "end_time": 1148882.606479056, + "total_evaluation_time_seconds": "218.57581721805036" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_16/boolq_2025-12-04T19-28-18.959993.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_16/boolq_2025-12-04T19-28-18.959993.json new file mode 100644 index 0000000000000000000000000000000000000000..bf95dc9e3472570cb4dbb2bd0a973c0c6cce892e --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_16/boolq_2025-12-04T19-28-18.959993.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7532110091743119, + "acc_stderr,none": 0.00754073641425668 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764847588.0141492, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1148468.696153889, + "end_time": 1148613.118341971, + "total_evaluation_time_seconds": "144.42218808201142" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_16/hellaswag_2025-12-04T19-52-25.664425.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_16/hellaswag_2025-12-04T19-52-25.664425.json new file mode 100644 index 0000000000000000000000000000000000000000..a4d8c93c41157454de1cd948ec49992f7690f52e --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_16/hellaswag_2025-12-04T19-52-25.664425.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.49283011352320255, + "acc_stderr,none": 0.004989268362968715, + "acc_norm,none": 0.6971718781119299, + "acc_norm_stderr,none": 0.004585424513012099 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764848359.0591547, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1149239.157679633, + "end_time": 1150059.822689351, + "total_evaluation_time_seconds": "820.6650097179227" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_16/piqa_2025-12-04T19-37-55.677195.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_16/piqa_2025-12-04T19-37-55.677195.json new file mode 100644 index 0000000000000000000000000000000000000000..b7f64355ab64b513c5c7be3de16f4e22ec32be8d --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_16/piqa_2025-12-04T19-37-55.677195.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7546245919477693, + "acc_stderr,none": 0.010039831320422393, + "acc_norm,none": 0.7698585418933623, + "acc_norm_stderr,none": 0.009820832826839813 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764848220.0224977, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1149099.224359257, + "end_time": 1149189.835511891, + "total_evaluation_time_seconds": "90.61115263402462" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_16/truthfulqa_mc1_2025-12-04T19-35-34.076908.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_16/truthfulqa_mc1_2025-12-04T19-35-34.076908.json new file mode 100644 index 0000000000000000000000000000000000000000..286e79054e768882aee80502cf9ee340f7a96fcd --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_16/truthfulqa_mc1_2025-12-04T19-35-34.076908.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.3011015911872705, + "acc_stderr,none": 0.01605899902610062 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764848056.7193093, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1148934.4804446, + "end_time": 1149048.235163327, + "total_evaluation_time_seconds": "113.75471872696653" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_16/winogrande_2025-12-04T19-25-02.126701.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_16/winogrande_2025-12-04T19-25-02.126701.json new file mode 100644 index 0000000000000000000000000000000000000000..9c764f23d8e9295a1afefbf094acef5bf4cfa11e --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_16/winogrande_2025-12-04T19-25-02.126701.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.5864246250986582, + "acc_stderr,none": 0.013840971763195303 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764847412.5355942, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1148291.915647894, + "end_time": 1148416.284996836, + "total_evaluation_time_seconds": "124.36934894206934" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_17/arc_challenge_2025-12-04T20-08-13.197303.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_17/arc_challenge_2025-12-04T20-08-13.197303.json new file mode 100644 index 0000000000000000000000000000000000000000..64f88966c62389684fb3ede1f7f39fd7503d3b12 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_17/arc_challenge_2025-12-04T20-08-13.197303.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.48378839590443684, + "acc_stderr,none": 0.014603708567414938, + "acc_norm,none": 0.5503412969283277, + "acc_norm_stderr,none": 0.014537144444284736 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 51 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764849905.2766922, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1150786.886528224, + "end_time": 1151007.355563722, + "total_evaluation_time_seconds": "220.4690354980994" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_17/boolq_2025-12-04T20-03-41.466200.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_17/boolq_2025-12-04T20-03-41.466200.json new file mode 100644 index 0000000000000000000000000000000000000000..ed5bec991c35696f42c5a63fea02d03d17812176 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_17/boolq_2025-12-04T20-03-41.466200.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7412844036697248, + "acc_stderr,none": 0.007659426910763684 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764849680.0600436, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1150559.521364469, + "end_time": 1150735.624649503, + "total_evaluation_time_seconds": "176.1032850339543" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_17/hellaswag_2025-12-04T20-28-32.849299.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_17/hellaswag_2025-12-04T20-28-32.849299.json new file mode 100644 index 0000000000000000000000000000000000000000..d8ce7c3fc9b1d91c25c2f8d434fa67231072a4ce --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_17/hellaswag_2025-12-04T20-28-32.849299.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.4955188209520016, + "acc_stderr,none": 0.0049895810081631905, + "acc_norm,none": 0.6996614220274846, + "acc_norm_stderr,none": 0.004574683373821045 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764850505.1464703, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1151385.598980608, + "end_time": 1152227.00740603, + "total_evaluation_time_seconds": "841.4084254221525" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_17/piqa_2025-12-04T20-13-39.797062.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_17/piqa_2025-12-04T20-13-39.797062.json new file mode 100644 index 0000000000000000000000000000000000000000..e296c2ff60e5e8acb50f83a578b05a1c047b24f1 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_17/piqa_2025-12-04T20-13-39.797062.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7513601741022851, + "acc_stderr,none": 0.010084511234296862, + "acc_norm,none": 0.7616974972796517, + "acc_norm_stderr,none": 0.009940334245876209 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764850354.3808591, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1151233.062205659, + "end_time": 1151333.955493222, + "total_evaluation_time_seconds": "100.89328756299801" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_17/truthfulqa_mc1_2025-12-04T20-11-07.352276.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_17/truthfulqa_mc1_2025-12-04T20-11-07.352276.json new file mode 100644 index 0000000000000000000000000000000000000000..2765606002bafb01a0f6dd8010eed066b3a7b02c --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_17/truthfulqa_mc1_2025-12-04T20-11-07.352276.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.3463892288861689, + "acc_stderr,none": 0.01665699710912515 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764850182.4535909, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1151060.53693574, + "end_time": 1151181.510567392, + "total_evaluation_time_seconds": "120.97363165207207" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_17/winogrande_2025-12-04T19-59-51.940135.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_17/winogrande_2025-12-04T19-59-51.940135.json new file mode 100644 index 0000000000000000000000000000000000000000..badc140284cbc021bb4b47cf34407facae3a9e0b --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_17/winogrande_2025-12-04T19-59-51.940135.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.601420678768745, + "acc_stderr,none": 0.013760357176873836 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764849531.2059152, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1150410.674170558, + "end_time": 1150506.098433411, + "total_evaluation_time_seconds": "95.42426285287365" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_18/arc_challenge_2025-12-04T20-44-15.330835.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_18/arc_challenge_2025-12-04T20-44-15.330835.json new file mode 100644 index 0000000000000000000000000000000000000000..02d68bb199c6c2e40795d37cee5a400ca7b149be --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_18/arc_challenge_2025-12-04T20-44-15.330835.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.47440273037542663, + "acc_stderr,none": 0.014592230885298967, + "acc_norm,none": 0.5187713310580204, + "acc_norm_stderr,none": 0.014601090150633962 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 51 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764852077.2827356, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1152957.75072085, + "end_time": 1153169.48910504, + "total_evaluation_time_seconds": "211.7383841900155" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_18/boolq_2025-12-04T20-39-52.554160.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_18/boolq_2025-12-04T20-39-52.554160.json new file mode 100644 index 0000000000000000000000000000000000000000..d206da75419a4030acec26cfc694cdc4a052302b --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_18/boolq_2025-12-04T20-39-52.554160.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7382262996941896, + "acc_stderr,none": 0.007688653730439833 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764851873.1785924, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1152752.459382579, + "end_time": 1152906.712522099, + "total_evaluation_time_seconds": "154.253139520064" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_18/hellaswag_2025-12-04T21-03-47.025526.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_18/hellaswag_2025-12-04T21-03-47.025526.json new file mode 100644 index 0000000000000000000000000000000000000000..88ccc63a68423ec719ebbd871a574ca9a1f80d90 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_18/hellaswag_2025-12-04T21-03-47.025526.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.48735311690898225, + "acc_stderr,none": 0.004988184988344818, + "acc_norm,none": 0.6940848436566421, + "acc_norm_stderr,none": 0.004598522271041223 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764852631.1335554, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1153511.789315502, + "end_time": 1154341.183556485, + "total_evaluation_time_seconds": "829.3942409830634" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_18/piqa_2025-12-04T20-49-07.179382.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_18/piqa_2025-12-04T20-49-07.179382.json new file mode 100644 index 0000000000000000000000000000000000000000..c89149f0161801b5ae6fc9477ad23978a94cdec8 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_18/piqa_2025-12-04T20-49-07.179382.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7568008705114254, + "acc_stderr,none": 0.010009611953858927, + "acc_norm,none": 0.764961915125136, + "acc_norm_stderr,none": 0.009893146688805336 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764852495.9050171, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1153375.124519373, + "end_time": 1153461.337724938, + "total_evaluation_time_seconds": "86.21320556499995" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_18/truthfulqa_mc1_2025-12-04T20-46-49.728229.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_18/truthfulqa_mc1_2025-12-04T20-46-49.728229.json new file mode 100644 index 0000000000000000000000000000000000000000..e94d2860039a4d9d3e47629d8982c6636c2d9ab1 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_18/truthfulqa_mc1_2025-12-04T20-46-49.728229.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.29253365973072215, + "acc_stderr,none": 0.015925597445286165 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764852341.841327, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1153221.021110358, + "end_time": 1153323.886341916, + "total_evaluation_time_seconds": "102.86523155798204" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_18/winogrande_2025-12-04T20-36-26.631019.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_18/winogrande_2025-12-04T20-36-26.631019.json new file mode 100644 index 0000000000000000000000000000000000000000..d1ae4c9a66eaf861aaf4420d183784bd74343d6a --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_18/winogrande_2025-12-04T20-36-26.631019.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.5895816890292028, + "acc_stderr,none": 0.013825107120035861 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764851700.2055447, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1152579.264436614, + "end_time": 1152700.789363293, + "total_evaluation_time_seconds": "121.52492667897604" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_19/arc_challenge_2025-12-04T21-18-53.986068.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_19/arc_challenge_2025-12-04T21-18-53.986068.json new file mode 100644 index 0000000000000000000000000000000000000000..4059ffe08d330a8bf16f89c771159113ae869a3c --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_19/arc_challenge_2025-12-04T21-18-53.986068.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.48890784982935154, + "acc_stderr,none": 0.01460779491401305, + "acc_norm,none": 0.5443686006825939, + "acc_norm_stderr,none": 0.014553749939306863 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 51 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764854159.3158426, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1155039.727178088, + "end_time": 1155248.144344656, + "total_evaluation_time_seconds": "208.41716656810604" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_19/boolq_2025-12-04T21-14-34.059474.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_19/boolq_2025-12-04T21-14-34.059474.json new file mode 100644 index 0000000000000000000000000000000000000000..faf4e3e1be521b274b3af288651b9b27fff4a349 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_19/boolq_2025-12-04T21-14-34.059474.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7452599388379205, + "acc_stderr,none": 0.007620703281690058 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764853960.3007696, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1154840.882850492, + "end_time": 1154988.217744381, + "total_evaluation_time_seconds": "147.33489388902672" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_19/hellaswag_2025-12-04T21-38-26.729753.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_19/hellaswag_2025-12-04T21-38-26.729753.json new file mode 100644 index 0000000000000000000000000000000000000000..3d177e240348572ad36bba849efee9e97db1fb23 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_19/hellaswag_2025-12-04T21-38-26.729753.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.492531368253336, + "acc_stderr,none": 0.004989224715784534, + "acc_norm,none": 0.7061342362079267, + "acc_norm_stderr,none": 0.004546002255456779 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764854709.4713848, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1155586.681185082, + "end_time": 1156420.887886944, + "total_evaluation_time_seconds": "834.2067018619273" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_19/piqa_2025-12-04T21-23-40.225621.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_19/piqa_2025-12-04T21-23-40.225621.json new file mode 100644 index 0000000000000000000000000000000000000000..4c58c4785a4689637d5aac1b6350dfe9463b0779 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_19/piqa_2025-12-04T21-23-40.225621.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.750816104461371, + "acc_stderr,none": 0.010091882770120214, + "acc_norm,none": 0.7633297062023939, + "acc_norm_stderr,none": 0.009916841655042806 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764854572.4376273, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1155452.649384735, + "end_time": 1155534.383917101, + "total_evaluation_time_seconds": "81.7345323660411" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_19/truthfulqa_mc1_2025-12-04T21-21-26.966024.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_19/truthfulqa_mc1_2025-12-04T21-21-26.966024.json new file mode 100644 index 0000000000000000000000000000000000000000..a5e17e359c603fedaad89012ac8557e573987a60 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_19/truthfulqa_mc1_2025-12-04T21-21-26.966024.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.33659730722154224, + "acc_stderr,none": 0.01654241280949488 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764854419.4130833, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1155299.250387258, + "end_time": 1155401.124129509, + "total_evaluation_time_seconds": "101.87374225119129" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_19/winogrande_2025-12-04T21-11-15.567663.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_19/winogrande_2025-12-04T21-11-15.567663.json new file mode 100644 index 0000000000000000000000000000000000000000..1a54686e25f49d2965b86ba84db972b6101f91d5 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_19/winogrande_2025-12-04T21-11-15.567663.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.584846093133386, + "acc_stderr,none": 0.013848684086658587 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764853815.2424896, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1154693.755069242, + "end_time": 1154789.725847076, + "total_evaluation_time_seconds": "95.97077783383429" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_2/arc_challenge_2025-12-04T11-32-09.306509.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_2/arc_challenge_2025-12-04T11-32-09.306509.json new file mode 100644 index 0000000000000000000000000000000000000000..f2bed60f4262d7e0500df4e45101b950a7a7d117 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_2/arc_challenge_2025-12-04T11-32-09.306509.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.447098976109215, + "acc_stderr,none": 0.014529380160526842, + "acc_norm,none": 0.5110921501706485, + "acc_norm_stderr,none": 0.014607794914013057 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 51 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764818963.543401, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1119844.590812882, + "end_time": 1120043.464789639, + "total_evaluation_time_seconds": "198.87397675705142" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_2/boolq_2025-12-04T11-28-00.938598.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_2/boolq_2025-12-04T11-28-00.938598.json new file mode 100644 index 0000000000000000000000000000000000000000..436df84b8122b03915456a10b8081400ffd7fb0a --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_2/boolq_2025-12-04T11-28-00.938598.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7217125382262997, + "acc_stderr,none": 0.007838292838766742 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764818774.619274, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1119655.289230176, + "end_time": 1119795.096880438, + "total_evaluation_time_seconds": "139.80765026202425" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_2/hellaswag_2025-12-04T11-51-30.015462.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_2/hellaswag_2025-12-04T11-51-30.015462.json new file mode 100644 index 0000000000000000000000000000000000000000..ef6da51946b14ccab131d712d1a804429ed2a2a6 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_2/hellaswag_2025-12-04T11-51-30.015462.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.4772953594901414, + "acc_stderr,none": 0.004984634285101616, + "acc_norm,none": 0.678550089623581, + "acc_norm_stderr,none": 0.0046607856169337755 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764819481.9088707, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1120363.243278305, + "end_time": 1121204.173526029, + "total_evaluation_time_seconds": "840.9302477240562" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_2/piqa_2025-12-04T11-36-37.804609.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_2/piqa_2025-12-04T11-36-37.804609.json new file mode 100644 index 0000000000000000000000000000000000000000..55fa73f9248d9112da0ee4d5215f275303edb6ce --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_2/piqa_2025-12-04T11-36-37.804609.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7252448313384113, + "acc_stderr,none": 0.010415033676676051, + "acc_norm,none": 0.7257889009793254, + "acc_norm_stderr,none": 0.01040861866493338 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764819354.5285106, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1120235.339735904, + "end_time": 1120311.962864858, + "total_evaluation_time_seconds": "76.62312895408832" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_2/truthfulqa_mc1_2025-12-04T11-34-30.437616.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_2/truthfulqa_mc1_2025-12-04T11-34-30.437616.json new file mode 100644 index 0000000000000000000000000000000000000000..66db0a75780b3361ec3a70699b41bfb78a7a6dd9 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_2/truthfulqa_mc1_2025-12-04T11-34-30.437616.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.2533659730722154, + "acc_stderr,none": 0.015225899340826814 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764819213.3471453, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1120094.260116363, + "end_time": 1120184.595908813, + "total_evaluation_time_seconds": "90.33579245000146" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_2/winogrande_2025-12-04T11-24-49.546472.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_2/winogrande_2025-12-04T11-24-49.546472.json new file mode 100644 index 0000000000000000000000000000000000000000..1e02d5544c158fc811ce4821c97b723fad838841 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_2/winogrande_2025-12-04T11-24-49.546472.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.6322020520915549, + "acc_stderr,none": 0.013552385559833596 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764818639.8075678, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1119518.639875715, + "end_time": 1119603.704748507, + "total_evaluation_time_seconds": "85.06487279199064" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_20/arc_challenge_2025-12-04T21-53-48.698290.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_20/arc_challenge_2025-12-04T21-53-48.698290.json new file mode 100644 index 0000000000000000000000000000000000000000..40e0aeff140230881e1905d560fc1d9e7e748c5f --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_20/arc_challenge_2025-12-04T21-53-48.698290.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.47696245733788395, + "acc_stderr,none": 0.014595873205358269, + "acc_norm,none": 0.515358361774744, + "acc_norm_stderr,none": 0.014604496129394904 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 51 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764856243.7979732, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1157124.241482747, + "end_time": 1157342.856521605, + "total_evaluation_time_seconds": "218.61503885802813" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_20/boolq_2025-12-04T21-49-18.849710.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_20/boolq_2025-12-04T21-49-18.849710.json new file mode 100644 index 0000000000000000000000000000000000000000..ffcdf5116ce6598bdea1c43648c66b4cf25fd551 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_20/boolq_2025-12-04T21-49-18.849710.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7412844036697248, + "acc_stderr,none": 0.007659426910763687 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764856045.2538314, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1156923.779874585, + "end_time": 1157073.008150022, + "total_evaluation_time_seconds": "149.228275436908" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_20/hellaswag_2025-12-04T22-13-21.436095.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_20/hellaswag_2025-12-04T22-13-21.436095.json new file mode 100644 index 0000000000000000000000000000000000000000..143a995ba3448bcae0e1fc253dfa762ba9ab52e2 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_20/hellaswag_2025-12-04T22-13-21.436095.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.47988448516231824, + "acc_stderr,none": 0.0049857417063857296, + "acc_norm,none": 0.6896036646086438, + "acc_norm_stderr,none": 0.0046171032803720155 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764856802.7903218, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1157682.519374218, + "end_time": 1158515.594364278, + "total_evaluation_time_seconds": "833.0749900599476" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_20/piqa_2025-12-04T21-58-36.248586.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_20/piqa_2025-12-04T21-58-36.248586.json new file mode 100644 index 0000000000000000000000000000000000000000..7ed02fe5b99085400ac4614fdd5ca7fa1724fb49 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_20/piqa_2025-12-04T21-58-36.248586.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7464635473340587, + "acc_stderr,none": 0.010150090834551786, + "acc_norm,none": 0.7600652883569097, + "acc_norm_stderr,none": 0.009963625892809545 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764856667.3426394, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1157545.223817852, + "end_time": 1157630.406923814, + "total_evaluation_time_seconds": "85.1831059618853" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_20/truthfulqa_mc1_2025-12-04T21-56-19.870320.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_20/truthfulqa_mc1_2025-12-04T21-56-19.870320.json new file mode 100644 index 0000000000000000000000000000000000000000..089054240d653be6851f8aad3525bd48fa31cb26 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_20/truthfulqa_mc1_2025-12-04T21-56-19.870320.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.31334149326805383, + "acc_stderr,none": 0.0162380650690596 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764856515.726712, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1157395.038996151, + "end_time": 1157494.028703154, + "total_evaluation_time_seconds": "98.98970700311475" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_20/winogrande_2025-12-04T21-45-57.762364.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_20/winogrande_2025-12-04T21-45-57.762364.json new file mode 100644 index 0000000000000000000000000000000000000000..51ceaff55459240e2273994ddbf2f8b65a02387f --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_20/winogrande_2025-12-04T21-45-57.762364.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.5887924230465666, + "acc_stderr,none": 0.01382912835867687 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764855894.7819893, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1156775.5824292, + "end_time": 1156871.920722743, + "total_evaluation_time_seconds": "96.33829354308546" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_21/arc_challenge_2025-12-04T22-28-47.441583.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_21/arc_challenge_2025-12-04T22-28-47.441583.json new file mode 100644 index 0000000000000000000000000000000000000000..72f3af5035c499a366e1d19ee764c4b46f0a3903 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_21/arc_challenge_2025-12-04T22-28-47.441583.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.46501706484641636, + "acc_stderr,none": 0.014575583922019667, + "acc_norm,none": 0.5093856655290102, + "acc_norm_stderr,none": 0.014608816322065 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 51 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764858338.4210615, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1159219.474571887, + "end_time": 1159441.599954405, + "total_evaluation_time_seconds": "222.12538251816295" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_21/boolq_2025-12-04T22-24-13.774272.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_21/boolq_2025-12-04T22-24-13.774272.json new file mode 100644 index 0000000000000000000000000000000000000000..b941196af4f2c3d5104ab85d225cad7aa335f341 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_21/boolq_2025-12-04T22-24-13.774272.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7394495412844037, + "acc_stderr,none": 0.00767702107251117 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764858132.068986, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1159012.102540251, + "end_time": 1159167.932711903, + "total_evaluation_time_seconds": "155.8301716519054" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_21/hellaswag_2025-12-04T22-48-37.414483.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_21/hellaswag_2025-12-04T22-48-37.414483.json new file mode 100644 index 0000000000000000000000000000000000000000..90def04a33d7151e1b6493fae35d525a11107fb9 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_21/hellaswag_2025-12-04T22-48-37.414483.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.46614220274845647, + "acc_stderr,none": 0.004978328190775522, + "acc_norm,none": 0.6796454889464251, + "acc_norm_stderr,none": 0.004656591678606725 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764858911.3210087, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1159790.402351688, + "end_time": 1160631.572478686, + "total_evaluation_time_seconds": "841.1701269980986" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_21/piqa_2025-12-04T22-33-45.451918.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_21/piqa_2025-12-04T22-33-45.451918.json new file mode 100644 index 0000000000000000000000000000000000000000..3333a84ba48a932d5f74d62601fa20db234e805b --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_21/piqa_2025-12-04T22-33-45.451918.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7415669205658324, + "acc_stderr,none": 0.01021397163677331, + "acc_norm,none": 0.7470076169749728, + "acc_norm_stderr,none": 0.010142888698862451 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764858775.793653, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1159655.54761603, + "end_time": 1159739.61025313, + "total_evaluation_time_seconds": "84.06263709999621" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_21/truthfulqa_mc1_2025-12-04T22-31-30.861423.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_21/truthfulqa_mc1_2025-12-04T22-31-30.861423.json new file mode 100644 index 0000000000000000000000000000000000000000..4c94f303d52110a6a342fc151994ba71852ab6f7 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_21/truthfulqa_mc1_2025-12-04T22-31-30.861423.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.3292533659730722, + "acc_stderr,none": 0.016451264440068242 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764858613.7186427, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1159493.179696555, + "end_time": 1159605.019785953, + "total_evaluation_time_seconds": "111.84008939797059" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_21/winogrande_2025-12-04T22-20-46.274871.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_21/winogrande_2025-12-04T22-20-46.274871.json new file mode 100644 index 0000000000000000000000000000000000000000..4f7f8546fd5ef091e8294f3d0a832d87c7dbf9a8 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_21/winogrande_2025-12-04T22-20-46.274871.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.5895816890292028, + "acc_stderr,none": 0.013825107120035868 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764857988.06292, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1158868.347692804, + "end_time": 1158960.433181156, + "total_evaluation_time_seconds": "92.08548835199326" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_22/arc_challenge_2025-12-04T23-04-44.968805.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_22/arc_challenge_2025-12-04T23-04-44.968805.json new file mode 100644 index 0000000000000000000000000000000000000000..1dc74ea4d2d5fb4b999353c8754339a732843a19 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_22/arc_challenge_2025-12-04T23-04-44.968805.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.4564846416382253, + "acc_stderr,none": 0.014555949760496439, + "acc_norm,none": 0.5025597269624573, + "acc_norm_stderr,none": 0.014611199329843791 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 51 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764860505.294705, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1161385.302233326, + "end_time": 1161599.127141074, + "total_evaluation_time_seconds": "213.82490774802864" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_22/boolq_2025-12-04T23-00-20.283469.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_22/boolq_2025-12-04T23-00-20.283469.json new file mode 100644 index 0000000000000000000000000000000000000000..7f7cf98fe7f6be6d52a375e7700373121496f924 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_22/boolq_2025-12-04T23-00-20.283469.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7342507645259939, + "acc_stderr,none": 0.007725929757288679 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764860270.2528114, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1161149.937467296, + "end_time": 1161334.441931916, + "total_evaluation_time_seconds": "184.50446462002583" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_22/hellaswag_2025-12-04T23-24-27.413478.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_22/hellaswag_2025-12-04T23-24-27.413478.json new file mode 100644 index 0000000000000000000000000000000000000000..eea7e2f9e5ecd0ed5e29923560ac6c7400e2f553 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_22/hellaswag_2025-12-04T23-24-27.413478.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.46285600477992433, + "acc_stderr,none": 0.004975993795562021, + "acc_norm,none": 0.6701852220673172, + "acc_norm_stderr,none": 0.004691848665399068 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764861071.01223, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1161950.275273225, + "end_time": 1162781.571576431, + "total_evaluation_time_seconds": "831.2963032058906" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_22/piqa_2025-12-04T23-09-44.969353.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_22/piqa_2025-12-04T23-09-44.969353.json new file mode 100644 index 0000000000000000000000000000000000000000..ad147b8744bd5ab967055bdca82d744e69b086c7 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_22/piqa_2025-12-04T23-09-44.969353.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7410228509249184, + "acc_stderr,none": 0.010220966031405605, + "acc_norm,none": 0.7431991294885746, + "acc_norm_stderr,none": 0.010192864802278054 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764860925.6353953, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1161805.836390965, + "end_time": 1161899.12767384, + "total_evaluation_time_seconds": "93.2912828749977" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_22/truthfulqa_mc1_2025-12-04T23-07-21.007636.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_22/truthfulqa_mc1_2025-12-04T23-07-21.007636.json new file mode 100644 index 0000000000000000000000000000000000000000..bd01526a378753a3992f20b964d562e384fcd388 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_22/truthfulqa_mc1_2025-12-04T23-07-21.007636.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.3378212974296206, + "acc_stderr,none": 0.016557167322516882 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764860770.3714392, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1161651.328361214, + "end_time": 1161755.165444605, + "total_evaluation_time_seconds": "103.83708339114673" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_22/winogrande_2025-12-04T22-56-19.532998.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_22/winogrande_2025-12-04T22-56-19.532998.json new file mode 100644 index 0000000000000000000000000000000000000000..22923dc5ed78ea229121802b5c9f4e79b6423b12 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_22/winogrande_2025-12-04T22-56-19.532998.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.5887924230465666, + "acc_stderr,none": 0.01382912835867687 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764860110.1655428, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1160986.975685728, + "end_time": 1161093.691227861, + "total_evaluation_time_seconds": "106.71554213296622" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_23/arc_challenge_2025-12-04T23-39-37.813105.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_23/arc_challenge_2025-12-04T23-39-37.813105.json new file mode 100644 index 0000000000000000000000000000000000000000..7eb3b6ca4b2654c75495eb5521ee78bcbcefc0a2 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_23/arc_challenge_2025-12-04T23-39-37.813105.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.4641638225255973, + "acc_stderr,none": 0.014573813664735714, + "acc_norm,none": 0.49658703071672355, + "acc_norm_stderr,none": 0.01461105040324407 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 51 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764862590.611816, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1163471.061751181, + "end_time": 1163691.971444882, + "total_evaluation_time_seconds": "220.9096937009599" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_23/boolq_2025-12-04T23-35-06.972045.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_23/boolq_2025-12-04T23-35-06.972045.json new file mode 100644 index 0000000000000000000000000000000000000000..d1db68c4486d94b89fec70e6ae749ab583ede38d --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_23/boolq_2025-12-04T23-35-06.972045.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.6700305810397553, + "acc_stderr,none": 0.008223878741654847 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764862394.686044, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1163274.714475949, + "end_time": 1163421.13020649, + "total_evaluation_time_seconds": "146.41573054087348" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_23/hellaswag_2025-12-04T23-59-04.872793.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_23/hellaswag_2025-12-04T23-59-04.872793.json new file mode 100644 index 0000000000000000000000000000000000000000..6b07fc6afdabffcdaf613a8269feb3a034a6ac17 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_23/hellaswag_2025-12-04T23-59-04.872793.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.4735112527384983, + "acc_stderr,none": 0.004982774293927781, + "acc_norm,none": 0.6778530173272257, + "acc_norm_stderr,none": 0.004663439181149013 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764863150.157231, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1164030.65003941, + "end_time": 1164859.030912872, + "total_evaluation_time_seconds": "828.3808734619524" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_23/piqa_2025-12-04T23-44-25.818583.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_23/piqa_2025-12-04T23-44-25.818583.json new file mode 100644 index 0000000000000000000000000000000000000000..14ffdcc879287d45dd62126a7fc0ac22bb89fb53 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_23/piqa_2025-12-04T23-44-25.818583.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.736126224156692, + "acc_stderr,none": 0.010282996367695568, + "acc_norm,none": 0.7546245919477693, + "acc_norm_stderr,none": 0.010039831320422396 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764863019.8504438, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1163899.41215175, + "end_time": 1163979.976911859, + "total_evaluation_time_seconds": "80.56476010894403" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_23/truthfulqa_mc1_2025-12-04T23-42-15.292968.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_23/truthfulqa_mc1_2025-12-04T23-42-15.292968.json new file mode 100644 index 0000000000000000000000000000000000000000..af9f72e201a088a473436314b0ba1920609484bc --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_23/truthfulqa_mc1_2025-12-04T23-42-15.292968.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.3268053855569155, + "acc_stderr,none": 0.016419874731135032 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764862862.7045147, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1163742.333985512, + "end_time": 1163849.451174146, + "total_evaluation_time_seconds": "107.11718863411807" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_23/winogrande_2025-12-04T23-31-50.342830.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_23/winogrande_2025-12-04T23-31-50.342830.json new file mode 100644 index 0000000000000000000000000000000000000000..e1a9f0d87aabe263ba273b0a06263fd899e57a39 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_23/winogrande_2025-12-04T23-31-50.342830.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.5951065509076559, + "acc_stderr,none": 0.013795927003124932 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764862255.4049842, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1163134.340791249, + "end_time": 1163224.501169077, + "total_evaluation_time_seconds": "90.16037782793865" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_24/arc_challenge_2025-12-05T00-13-52.810759.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_24/arc_challenge_2025-12-05T00-13-52.810759.json new file mode 100644 index 0000000000000000000000000000000000000000..447faa6c4a0cb43c7c5c4c2245b2ae547c296382 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_24/arc_challenge_2025-12-05T00-13-52.810759.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.4513651877133106, + "acc_stderr,none": 0.014542104569955258, + "acc_norm,none": 0.49829351535836175, + "acc_norm_stderr,none": 0.014611305705056976 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 51 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764864663.2720091, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1165543.126045469, + "end_time": 1165746.969018365, + "total_evaluation_time_seconds": "203.84297289606184" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_24/boolq_2025-12-05T00-09-37.756178.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_24/boolq_2025-12-05T00-09-37.756178.json new file mode 100644 index 0000000000000000000000000000000000000000..89b2de81d3ae433c5ba4ef4a23977e11ee721bd2 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_24/boolq_2025-12-05T00-09-37.756178.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7064220183486238, + "acc_stderr,none": 0.007965011249420069 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764864468.7851145, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1165348.712590258, + "end_time": 1165491.914471908, + "total_evaluation_time_seconds": "143.20188164990395" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_24/hellaswag_2025-12-05T00-33-09.896686.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_24/hellaswag_2025-12-05T00-33-09.896686.json new file mode 100644 index 0000000000000000000000000000000000000000..86b8f56e760876a3a0809bd16a4979c19e9e4a49 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_24/hellaswag_2025-12-05T00-33-09.896686.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.48536148177653854, + "acc_stderr,none": 0.004987642470249517, + "acc_norm,none": 0.6900019916351324, + "acc_norm_stderr,none": 0.004615472210316036 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764865197.057317, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1166076.907251923, + "end_time": 1166904.054492051, + "total_evaluation_time_seconds": "827.1472401279025" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_24/piqa_2025-12-05T00-18-30.814505.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_24/piqa_2025-12-05T00-18-30.814505.json new file mode 100644 index 0000000000000000000000000000000000000000..174cc05fb495921a125e3b5c388601bcf0b57efa --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_24/piqa_2025-12-05T00-18-30.814505.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.735582154515778, + "acc_stderr,none": 0.010289787244767158, + "acc_norm,none": 0.7568008705114254, + "acc_norm_stderr,none": 0.010009611953858931 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764865063.6530774, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1165943.236272081, + "end_time": 1166024.972793024, + "total_evaluation_time_seconds": "81.73652094299905" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_24/truthfulqa_mc1_2025-12-05T00-16-18.078249.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_24/truthfulqa_mc1_2025-12-05T00-16-18.078249.json new file mode 100644 index 0000000000000000000000000000000000000000..23e0f1f8b090dd8856c8bc379255edf4d86cb352 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_24/truthfulqa_mc1_2025-12-05T00-16-18.078249.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.3072215422276622, + "acc_stderr,none": 0.016150201321323002 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764864915.6566284, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1165797.175787774, + "end_time": 1165892.236580569, + "total_evaluation_time_seconds": "95.06079279514961" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_24/winogrande_2025-12-05T00-06-23.941554.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_24/winogrande_2025-12-05T00-06-23.941554.json new file mode 100644 index 0000000000000000000000000000000000000000..6d450b34ba1afe41a2434fbaa339e1026e36655a --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_24/winogrande_2025-12-05T00-06-23.941554.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.6045777426992897, + "acc_stderr,none": 0.013741678387545345 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764864331.1858754, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1165209.339581755, + "end_time": 1165298.099871751, + "total_evaluation_time_seconds": "88.76028999593109" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_25/arc_challenge_2025-12-05T00-47-44.258956.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_25/arc_challenge_2025-12-05T00-47-44.258956.json new file mode 100644 index 0000000000000000000000000000000000000000..eb82df0dbbebe27791ee92e60f21b4d4a4eec786 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_25/arc_challenge_2025-12-05T00-47-44.258956.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.48378839590443684, + "acc_stderr,none": 0.014603708567414941, + "acc_norm,none": 0.5315699658703071, + "acc_norm_stderr,none": 0.014582236460866968 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 51 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764866694.547262, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1167574.318605497, + "end_time": 1167778.417277388, + "total_evaluation_time_seconds": "204.09867189102806" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_25/boolq_2025-12-05T00-43-29.365710.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_25/boolq_2025-12-05T00-43-29.365710.json new file mode 100644 index 0000000000000000000000000000000000000000..d23364e4b9e24f608ec0ae986bda8eb3e50b44d2 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_25/boolq_2025-12-05T00-43-29.365710.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7311926605504587, + "acc_stderr,none": 0.0077540574189838675 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764866506.0186844, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1167387.678734639, + "end_time": 1167523.523999077, + "total_evaluation_time_seconds": "135.84526443807408" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_25/hellaswag_2025-12-05T01-06-55.143972.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_25/hellaswag_2025-12-05T01-06-55.143972.json new file mode 100644 index 0000000000000000000000000000000000000000..de54f41d8e7e4b1c71e800265f2bcd95b05ec520 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_25/hellaswag_2025-12-05T01-06-55.143972.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.5060744871539534, + "acc_stderr,none": 0.0049894131580348056, + "acc_norm,none": 0.7059350726946824, + "acc_norm_stderr,none": 0.004546901132945126 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764867223.5127187, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1168105.338539433, + "end_time": 1168929.302257827, + "total_evaluation_time_seconds": "823.9637183940504" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_25/piqa_2025-12-05T00-52-20.515427.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_25/piqa_2025-12-05T00-52-20.515427.json new file mode 100644 index 0000000000000000000000000000000000000000..2b489169c6522d9250345a60ef99e3c72ffe3f5c --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_25/piqa_2025-12-05T00-52-20.515427.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7421109902067464, + "acc_stderr,none": 0.010206956662056255, + "acc_norm,none": 0.7622415669205659, + "acc_norm_stderr,none": 0.009932525779525485 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764867097.3882694, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1167978.057177568, + "end_time": 1168054.67373592, + "total_evaluation_time_seconds": "76.61655835201964" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_25/truthfulqa_mc1_2025-12-05T00-50-13.426656.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_25/truthfulqa_mc1_2025-12-05T00-50-13.426656.json new file mode 100644 index 0000000000000000000000000000000000000000..6ea9157b63b464828dd9c0de0daee38862949ed4 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_25/truthfulqa_mc1_2025-12-05T00-50-13.426656.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.3011015911872705, + "acc_stderr,none": 0.01605899902610061 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764866947.9483426, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1167828.877890444, + "end_time": 1167927.584556357, + "total_evaluation_time_seconds": "98.70666591310874" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_25/winogrande_2025-12-05T00-40-23.620470.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_25/winogrande_2025-12-05T00-40-23.620470.json new file mode 100644 index 0000000000000000000000000000000000000000..946b92f1bf85e6e016a948fe86081d9c2d0de349 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_25/winogrande_2025-12-05T00-40-23.620470.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.6101026045777427, + "acc_stderr,none": 0.01370754731700847 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764866373.135642, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1167254.01346302, + "end_time": 1167337.778823798, + "total_evaluation_time_seconds": "83.76536077796482" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_26/arc_challenge_2025-12-05T01-21-35.571061.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_26/arc_challenge_2025-12-05T01-21-35.571061.json new file mode 100644 index 0000000000000000000000000000000000000000..c89e2242643fba0aa151f06dcfc3ae2fd7984ed3 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_26/arc_challenge_2025-12-05T01-21-35.571061.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.5247440273037542, + "acc_stderr,none": 0.01459348769493774, + "acc_norm,none": 0.5631399317406144, + "acc_norm_stderr,none": 0.01449442158425652 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 51 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764868725.860274, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1169606.154497597, + "end_time": 1169809.72935444, + "total_evaluation_time_seconds": "203.57485684310086" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_26/boolq_2025-12-05T01-17-20.114454.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_26/boolq_2025-12-05T01-17-20.114454.json new file mode 100644 index 0000000000000000000000000000000000000000..7be4f411f6477126c17b05b821734d144b48a4b9 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_26/boolq_2025-12-05T01-17-20.114454.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.6452599388379205, + "acc_stderr,none": 0.008367871633282294 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764868537.1456277, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1169417.772213411, + "end_time": 1169554.272677865, + "total_evaluation_time_seconds": "136.50046445406042" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_26/hellaswag_2025-12-05T01-40-32.028937.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_26/hellaswag_2025-12-05T01-40-32.028937.json new file mode 100644 index 0000000000000000000000000000000000000000..3616d9466556225856670565f877000da2e27fa8 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_26/hellaswag_2025-12-05T01-40-32.028937.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.5191196972714599, + "acc_stderr,none": 0.004986131919673968, + "acc_norm,none": 0.719577773351922, + "acc_norm_stderr,none": 0.004482874732237348 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764869244.4308753, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1170124.081491447, + "end_time": 1170946.187286623, + "total_evaluation_time_seconds": "822.1057951760013" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_26/piqa_2025-12-05T01-25-58.215618.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_26/piqa_2025-12-05T01-25-58.215618.json new file mode 100644 index 0000000000000000000000000000000000000000..c2cde354c36178105d51ba581780db9f6c08b8cd --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_26/piqa_2025-12-05T01-25-58.215618.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7475516866158868, + "acc_stderr,none": 0.01013566554736236, + "acc_norm,none": 0.7611534276387377, + "acc_norm_stderr,none": 0.0099481203853375 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764869115.8421376, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1169997.701897467, + "end_time": 1170072.373914729, + "total_evaluation_time_seconds": "74.67201726208441" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_26/truthfulqa_mc1_2025-12-05T01-23-53.005756.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_26/truthfulqa_mc1_2025-12-05T01-23-53.005756.json new file mode 100644 index 0000000000000000000000000000000000000000..458685a636802a97099f37e6207d8ad5c3fede60 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_26/truthfulqa_mc1_2025-12-05T01-23-53.005756.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.3157894736842105, + "acc_stderr,none": 0.016272287957916916 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764868976.7961912, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1169859.587415864, + "end_time": 1169947.164079913, + "total_evaluation_time_seconds": "87.57666404894553" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_26/winogrande_2025-12-05T01-14-12.670387.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_26/winogrande_2025-12-05T01-14-12.670387.json new file mode 100644 index 0000000000000000000000000000000000000000..ad7e508fb5796408bd8dbfca8d2171501ae4a92c --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_26/winogrande_2025-12-05T01-14-12.670387.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.6258879242304657, + "acc_stderr,none": 0.013599792958329821 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764868403.699633, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1169280.581051621, + "end_time": 1169366.828749992, + "total_evaluation_time_seconds": "86.24769837106578" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_27/arc_challenge_2025-12-05T01-55-02.366442.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_27/arc_challenge_2025-12-05T01-55-02.366442.json new file mode 100644 index 0000000000000000000000000000000000000000..449991b37466b0092081bb202064526a2d81b4ec --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_27/arc_challenge_2025-12-05T01-55-02.366442.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.5034129692832765, + "acc_stderr,none": 0.014611050403244084, + "acc_norm,none": 0.5435153583617748, + "acc_norm_stderr,none": 0.014555949760496446 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 51 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764870737.8153622, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1171619.016449187, + "end_time": 1171816.524719249, + "total_evaluation_time_seconds": "197.50827006204054" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_27/boolq_2025-12-05T01-50-53.777049.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_27/boolq_2025-12-05T01-50-53.777049.json new file mode 100644 index 0000000000000000000000000000000000000000..d8d83c242546788f03f7dae917f2b95f8c23ebda --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_27/boolq_2025-12-05T01-50-53.777049.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7486238532110092, + "acc_stderr,none": 0.007587285084733795 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764870551.67338, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1171431.604446895, + "end_time": 1171567.934996209, + "total_evaluation_time_seconds": "136.33054931415245" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_27/hellaswag_2025-12-05T02-13-52.546607.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_27/hellaswag_2025-12-05T02-13-52.546607.json new file mode 100644 index 0000000000000000000000000000000000000000..352b5aed256d927af5fa8dc1535d85e9693758b9 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_27/hellaswag_2025-12-05T02-13-52.546607.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.5336586337382991, + "acc_stderr,none": 0.004978462690966914, + "acc_norm,none": 0.7136028679545907, + "acc_norm_stderr,none": 0.004511533039406222 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764871246.8814569, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1172128.961382795, + "end_time": 1172946.704965173, + "total_evaluation_time_seconds": "817.7435823781416" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_27/piqa_2025-12-05T01-59-25.709592.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_27/piqa_2025-12-05T01-59-25.709592.json new file mode 100644 index 0000000000000000000000000000000000000000..8b856e2d04e4c70f913d4541db235007e04255cd --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_27/piqa_2025-12-05T01-59-25.709592.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7464635473340587, + "acc_stderr,none": 0.010150090834551793, + "acc_norm,none": 0.7584330794341676, + "acc_norm_stderr,none": 0.009986718001804484 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764871125.3714387, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1172007.450286689, + "end_time": 1172079.867946184, + "total_evaluation_time_seconds": "72.41765949502587" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_27/truthfulqa_mc1_2025-12-05T01-57-22.218169.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_27/truthfulqa_mc1_2025-12-05T01-57-22.218169.json new file mode 100644 index 0000000000000000000000000000000000000000..6932b426482c9dc1c4e761d6b9e6907bbd2ea669 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_27/truthfulqa_mc1_2025-12-05T01-57-22.218169.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.3525091799265606, + "acc_stderr,none": 0.016724646380756544 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764870985.7195861, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1171866.80311283, + "end_time": 1171956.376508467, + "total_evaluation_time_seconds": "89.573395636864" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_27/winogrande_2025-12-05T01-47-47.133877.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_27/winogrande_2025-12-05T01-47-47.133877.json new file mode 100644 index 0000000000000000000000000000000000000000..6a2368ea6edc6628191d394233c20b38ae90e04d --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_27/winogrande_2025-12-05T01-47-47.133877.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.595895816890292, + "acc_stderr,none": 0.013791610664670858 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764870417.51003, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1171298.838732449, + "end_time": 1171381.292170453, + "total_evaluation_time_seconds": "82.45343800378032" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_3/arc_challenge_2025-12-04T12-06-07.360394.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_3/arc_challenge_2025-12-04T12-06-07.360394.json new file mode 100644 index 0000000000000000000000000000000000000000..bd1946d0a53cb118616638e4d8c094409cdfb044 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_3/arc_challenge_2025-12-04T12-06-07.360394.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.20648464163822525, + "acc_stderr,none": 0.011828865619002316, + "acc_norm,none": 0.24573378839590443, + "acc_norm_stderr,none": 0.012581033453730107 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 51 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764821000.3916638, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1121879.555508534, + "end_time": 1122081.518770427, + "total_evaluation_time_seconds": "201.96326189301908" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_3/boolq_2025-12-04T12-01-53.832012.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_3/boolq_2025-12-04T12-01-53.832012.json new file mode 100644 index 0000000000000000000000000000000000000000..108523b5e95a3b18280100d7abd5c96224f05fdc --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_3/boolq_2025-12-04T12-01-53.832012.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.5929663608562691, + "acc_stderr,none": 0.00859256288706887 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764820814.7014577, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1121694.903304254, + "end_time": 1121827.990367823, + "total_evaluation_time_seconds": "133.0870635691099" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_3/hellaswag_2025-12-04T12-25-16.381396.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_3/hellaswag_2025-12-04T12-25-16.381396.json new file mode 100644 index 0000000000000000000000000000000000000000..be28667897d2fbfa3a44694d3ded8ad74c8e3da8 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_3/hellaswag_2025-12-04T12-25-16.381396.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.27922724556861184, + "acc_stderr,none": 0.004477025762200604, + "acc_norm,none": 0.30173272256522604, + "acc_norm_stderr,none": 0.004580718115992502 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764821528.1451025, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1122407.308767015, + "end_time": 1123230.539520676, + "total_evaluation_time_seconds": "823.2307536609005" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_3/piqa_2025-12-04T12-10-39.784807.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_3/piqa_2025-12-04T12-10-39.784807.json new file mode 100644 index 0000000000000000000000000000000000000000..bfc7370cd538c0ce85d62efb0ba6604f7a4203be --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_3/piqa_2025-12-04T12-10-39.784807.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.5946681175190425, + "acc_stderr,none": 0.011454816387346765, + "acc_norm,none": 0.5767138193688792, + "acc_norm_stderr,none": 0.011527699473614475 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764821397.6450157, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1122277.899828533, + "end_time": 1122353.943191974, + "total_evaluation_time_seconds": "76.04336344101466" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_3/truthfulqa_mc1_2025-12-04T12-08-32.243270.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_3/truthfulqa_mc1_2025-12-04T12-08-32.243270.json new file mode 100644 index 0000000000000000000000000000000000000000..6aa3a8cfb4301989d26f661763ac98f934005d21 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_3/truthfulqa_mc1_2025-12-04T12-08-32.243270.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.25703794369645044, + "acc_stderr,none": 0.015298077509485083 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764821254.4241943, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1122133.6635718, + "end_time": 1122226.401639737, + "total_evaluation_time_seconds": "92.73806793685071" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_3/winogrande_2025-12-04T11-58-49.913562.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_3/winogrande_2025-12-04T11-58-49.913562.json new file mode 100644 index 0000000000000000000000000000000000000000..19bd3849dfcaab4bcb2a1f705aeb01d43f88bfe9 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_3/winogrande_2025-12-04T11-58-49.913562.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.526440410418311, + "acc_stderr,none": 0.014032823874407222 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764820686.4372277, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1121566.554694095, + "end_time": 1121644.071924142, + "total_evaluation_time_seconds": "77.51723004714586" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_4/arc_challenge_2025-12-04T12-39-46.452692.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_4/arc_challenge_2025-12-04T12-39-46.452692.json new file mode 100644 index 0000000000000000000000000000000000000000..bdfb6814955402bf955593032a84e82e9a027161 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_4/arc_challenge_2025-12-04T12-39-46.452692.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.4761092150170648, + "acc_stderr,none": 0.014594701798071652, + "acc_norm,none": 0.5204778156996587, + "acc_norm_stderr,none": 0.014599131353035016 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 51 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764823019.7190943, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1123900.232217949, + "end_time": 1124100.611073013, + "total_evaluation_time_seconds": "200.37885506404564" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_4/boolq_2025-12-04T12-35-32.121868.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_4/boolq_2025-12-04T12-35-32.121868.json new file mode 100644 index 0000000000000000000000000000000000000000..4be2112b18119d0c2e907b81b0ffbafe7dcbae92 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_4/boolq_2025-12-04T12-35-32.121868.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7480122324159021, + "acc_stderr,none": 0.007593405967377497 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764822831.6146834, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1123713.948335013, + "end_time": 1123846.280230722, + "total_evaluation_time_seconds": "132.33189570903778" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_4/hellaswag_2025-12-04T12-59-02.697552.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_4/hellaswag_2025-12-04T12-59-02.697552.json new file mode 100644 index 0000000000000000000000000000000000000000..b61988e3d0f14010ad6917d5eef7dfb2eb1e84fa --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_4/hellaswag_2025-12-04T12-59-02.697552.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.47998406691894047, + "acc_stderr,none": 0.004985781620467018, + "acc_norm,none": 0.6755626369249154, + "acc_norm_stderr,none": 0.004672074496749045 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764823554.3687458, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1124433.593779207, + "end_time": 1125256.855865145, + "total_evaluation_time_seconds": "823.2620859381277" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_4/piqa_2025-12-04T12-44-26.757874.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_4/piqa_2025-12-04T12-44-26.757874.json new file mode 100644 index 0000000000000000000000000000000000000000..0471ce13fcce46bf9cd059a171a62794cc97d248 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_4/piqa_2025-12-04T12-44-26.757874.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.719804134929271, + "acc_stderr,none": 0.010478122015577084, + "acc_norm,none": 0.721436343852013, + "acc_norm_stderr,none": 0.010459397235965171 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764823423.410724, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1124304.657554529, + "end_time": 1124380.916136783, + "total_evaluation_time_seconds": "76.25858225417323" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_4/truthfulqa_mc1_2025-12-04T12-42-19.693565.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_4/truthfulqa_mc1_2025-12-04T12-42-19.693565.json new file mode 100644 index 0000000000000000000000000000000000000000..0d8df125cdae9e4c1c0e8f76f35bc61725bd4f1f --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_4/truthfulqa_mc1_2025-12-04T12-42-19.693565.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.26193390452876375, + "acc_stderr,none": 0.01539211880501501 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764823273.3031003, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1124151.922482065, + "end_time": 1124253.851736629, + "total_evaluation_time_seconds": "101.92925456399098" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_4/winogrande_2025-12-04T12-32-29.452881.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_4/winogrande_2025-12-04T12-32-29.452881.json new file mode 100644 index 0000000000000000000000000000000000000000..d61403d3def92b893479aa2a837fd0a80263dd5e --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_4/winogrande_2025-12-04T12-32-29.452881.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.6195737963693765, + "acc_stderr,none": 0.013644727908656833 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764822704.686668, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1123582.610900908, + "end_time": 1123663.611297653, + "total_evaluation_time_seconds": "81.0003967450466" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_5/arc_challenge_2025-12-04T13-13-34.306973.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_5/arc_challenge_2025-12-04T13-13-34.306973.json new file mode 100644 index 0000000000000000000000000000000000000000..e554cb3ef621c1e41b570ae0991f0ad223ad7175 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_5/arc_challenge_2025-12-04T13-13-34.306973.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.41638225255972694, + "acc_stderr,none": 0.014405618279436167, + "acc_norm,none": 0.46928327645051193, + "acc_norm_stderr,none": 0.01458379254630404 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 51 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764825048.479555, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1125928.698657643, + "end_time": 1126128.46533662, + "total_evaluation_time_seconds": "199.7666789770592" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_5/boolq_2025-12-04T13-09-23.170033.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_5/boolq_2025-12-04T13-09-23.170033.json new file mode 100644 index 0000000000000000000000000000000000000000..f6ae0e7c20f11896d4910fc6c01f76b4b9343f51 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_5/boolq_2025-12-04T13-09-23.170033.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7324159021406728, + "acc_stderr,none": 0.007742862953407332 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764824862.6324472, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1125743.903260105, + "end_time": 1125877.328253379, + "total_evaluation_time_seconds": "133.42499327403493" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_5/hellaswag_2025-12-04T13-32-41.369041.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_5/hellaswag_2025-12-04T13-32-41.369041.json new file mode 100644 index 0000000000000000000000000000000000000000..38eca84f68c03d4c6ee859942c3cf493c383bd13 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_5/hellaswag_2025-12-04T13-32-41.369041.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.469627564230233, + "acc_stderr,none": 0.004980566907790458, + "acc_norm,none": 0.6552479585739892, + "acc_norm_stderr,none": 0.0047431600342711525 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764825570.4483027, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1126449.890390508, + "end_time": 1127275.527380603, + "total_evaluation_time_seconds": "825.6369900947902" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_5/piqa_2025-12-04T13-18-03.652375.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_5/piqa_2025-12-04T13-18-03.652375.json new file mode 100644 index 0000000000000000000000000000000000000000..e11ab268fe2ab6a1623aa9194321b27fb04acbb5 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_5/piqa_2025-12-04T13-18-03.652375.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7225244831338411, + "acc_stderr,none": 0.010446818281039941, + "acc_norm,none": 0.7410228509249184, + "acc_norm_stderr,none": 0.010220966031405605 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764825441.0923066, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1126321.686907717, + "end_time": 1126397.810737762, + "total_evaluation_time_seconds": "76.12383004487492" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_5/truthfulqa_mc1_2025-12-04T13-15-54.696554.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_5/truthfulqa_mc1_2025-12-04T13-15-54.696554.json new file mode 100644 index 0000000000000000000000000000000000000000..477c8d3ae6179fedcf7dc959849a0c621c9908cc --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_5/truthfulqa_mc1_2025-12-04T13-15-54.696554.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.27539779681762544, + "acc_stderr,none": 0.015638135667775523 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764825300.4040918, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1126180.705570187, + "end_time": 1126268.854828553, + "total_evaluation_time_seconds": "88.14925836608745" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_5/winogrande_2025-12-04T13-06-18.807171.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_5/winogrande_2025-12-04T13-06-18.807171.json new file mode 100644 index 0000000000000000000000000000000000000000..959ac2a1b9a7a44bde1b02b44bf1768c819d0062 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_5/winogrande_2025-12-04T13-06-18.807171.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.5469613259668509, + "acc_stderr,none": 0.013990366632148091 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764824732.0582159, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1125610.396715083, + "end_time": 1125692.96547771, + "total_evaluation_time_seconds": "82.5687626269646" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_6/arc_challenge_2025-12-04T13-47-18.139939.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_6/arc_challenge_2025-12-04T13-47-18.139939.json new file mode 100644 index 0000000000000000000000000000000000000000..163c4c6aa815c3a006163e0fdaa9eb7224aeab20 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_6/arc_challenge_2025-12-04T13-47-18.139939.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.4863481228668942, + "acc_stderr,none": 0.014605943429860947, + "acc_norm,none": 0.5418088737201365, + "acc_norm_stderr,none": 0.014560220308714693 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 51 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764827066.9966152, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1127946.514942277, + "end_time": 1128152.298218876, + "total_evaluation_time_seconds": "205.78327659890056" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_6/boolq_2025-12-04T13-43-02.495801.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_6/boolq_2025-12-04T13-43-02.495801.json new file mode 100644 index 0000000000000000000000000000000000000000..8dfcf1906930abb1d0053e7c2684425323b5f485 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_6/boolq_2025-12-04T13-43-02.495801.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.735474006116208, + "acc_stderr,none": 0.007714546144910652 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764826880.2848723, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1127759.929604033, + "end_time": 1127896.654159893, + "total_evaluation_time_seconds": "136.7245558600407" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_6/hellaswag_2025-12-04T14-06-31.421650.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_6/hellaswag_2025-12-04T14-06-31.421650.json new file mode 100644 index 0000000000000000000000000000000000000000..e0bf64472c9ab053831c161c6f3a8c1981d654ae --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_6/hellaswag_2025-12-04T14-06-31.421650.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.4856602270464051, + "acc_stderr,none": 0.004987728900897601, + "acc_norm,none": 0.6868153754232225, + "acc_norm_stderr,none": 0.004628409084218778 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764827604.0514324, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1128486.025175133, + "end_time": 1129305.579990629, + "total_evaluation_time_seconds": "819.5548154960852" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_6/piqa_2025-12-04T13-52-01.431903.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_6/piqa_2025-12-04T13-52-01.431903.json new file mode 100644 index 0000000000000000000000000000000000000000..4c9e8c78e94bf1ec0ef8b815afeb9a9aa91218e0 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_6/piqa_2025-12-04T13-52-01.431903.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7388465723612623, + "acc_stderr,none": 0.010248738649935581, + "acc_norm,none": 0.750816104461371, + "acc_norm_stderr,none": 0.01009188277012022 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764827477.066381, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1128357.376420093, + "end_time": 1128435.590275921, + "total_evaluation_time_seconds": "78.21385582792573" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_6/truthfulqa_mc1_2025-12-04T13-49-49.406947.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_6/truthfulqa_mc1_2025-12-04T13-49-49.406947.json new file mode 100644 index 0000000000000000000000000000000000000000..07ab22752551d09f76e5981354148bc27868731c --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_6/truthfulqa_mc1_2025-12-04T13-49-49.406947.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.3011015911872705, + "acc_stderr,none": 0.016058999026100612 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764827328.3551626, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1128208.904958728, + "end_time": 1128303.565335583, + "total_evaluation_time_seconds": "94.66037685493939" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_6/winogrande_2025-12-04T13-39-51.930754.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_6/winogrande_2025-12-04T13-39-51.930754.json new file mode 100644 index 0000000000000000000000000000000000000000..c9fa1b351086593db8b103a5722c316eb57df709 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_6/winogrande_2025-12-04T13-39-51.930754.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.6195737963693765, + "acc_stderr,none": 0.013644727908656826 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764826745.2893574, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1127627.415544325, + "end_time": 1127706.089052911, + "total_evaluation_time_seconds": "78.67350858589634" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_7/arc_challenge_2025-12-04T14-21-46.518523.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_7/arc_challenge_2025-12-04T14-21-46.518523.json new file mode 100644 index 0000000000000000000000000000000000000000..7c7b6a0018c421ed31f9800fd5f4f3faca0b8281 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_7/arc_challenge_2025-12-04T14-21-46.518523.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.44368600682593856, + "acc_stderr,none": 0.014518421825670437, + "acc_norm,none": 0.4906143344709898, + "acc_norm_stderr,none": 0.014608816322065003 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 51 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764829129.863107, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1130008.96542831, + "end_time": 1130220.67682409, + "total_evaluation_time_seconds": "211.71139578009024" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_7/boolq_2025-12-04T14-17-14.867332.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_7/boolq_2025-12-04T14-17-14.867332.json new file mode 100644 index 0000000000000000000000000000000000000000..e89d16a64331e06badc9a390a1cbc9edc325c030 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_7/boolq_2025-12-04T14-17-14.867332.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7865443425076453, + "acc_stderr,none": 0.00716651444343289 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764828928.2056072, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1129806.363221076, + "end_time": 1129949.025727081, + "total_evaluation_time_seconds": "142.66250600502826" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_7/hellaswag_2025-12-04T14-41-12.963546.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_7/hellaswag_2025-12-04T14-41-12.963546.json new file mode 100644 index 0000000000000000000000000000000000000000..c675dabe3cb09c454efddbd67fd6dab348fbed08 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_7/hellaswag_2025-12-04T14-41-12.963546.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.47988448516231824, + "acc_stderr,none": 0.0049857417063857296, + "acc_norm,none": 0.686018721370245, + "acc_norm_stderr,none": 0.004631603539751971 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764829680.9014292, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1130560.980449967, + "end_time": 1131387.121691697, + "total_evaluation_time_seconds": "826.141241729958" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_7/piqa_2025-12-04T14-26-34.805749.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_7/piqa_2025-12-04T14-26-34.805749.json new file mode 100644 index 0000000000000000000000000000000000000000..62e1a1f85255d53d078140f3f6a1a50e07569640 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_7/piqa_2025-12-04T14-26-34.805749.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7372143634385201, + "acc_stderr,none": 0.010269354068140767, + "acc_norm,none": 0.7453754080522307, + "acc_norm_stderr,none": 0.010164432237060478 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764829544.1987987, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1130422.445635487, + "end_time": 1130508.964128184, + "total_evaluation_time_seconds": "86.51849269680679" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_7/truthfulqa_mc1_2025-12-04T14-24-16.774686.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_7/truthfulqa_mc1_2025-12-04T14-24-16.774686.json new file mode 100644 index 0000000000000000000000000000000000000000..300f1c77723a8376ad8db7429e2aa20338119fe8 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_7/truthfulqa_mc1_2025-12-04T14-24-16.774686.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.25703794369645044, + "acc_stderr,none": 0.015298077509485083 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764829393.0154648, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1130271.826527923, + "end_time": 1130370.93304113, + "total_evaluation_time_seconds": "99.10651320684701" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_7/winogrande_2025-12-04T14-14-00.664488.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_7/winogrande_2025-12-04T14-14-00.664488.json new file mode 100644 index 0000000000000000000000000000000000000000..67fe0fe5453f80af106860eb88e515114446fe3d --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_7/winogrande_2025-12-04T14-14-00.664488.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.569060773480663, + "acc_stderr,none": 0.013917796623335966 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764828779.6498911, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1129658.771180289, + "end_time": 1129754.822865989, + "total_evaluation_time_seconds": "96.05168569996022" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_8/arc_challenge_2025-12-04T14-56-23.830960.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_8/arc_challenge_2025-12-04T14-56-23.830960.json new file mode 100644 index 0000000000000000000000000000000000000000..afd573d67d7f2c00071ea3ffdffbb62f959ff027 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_8/arc_challenge_2025-12-04T14-56-23.830960.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.4598976109215017, + "acc_stderr,none": 0.014564318856924848, + "acc_norm,none": 0.5008532423208191, + "acc_norm_stderr,none": 0.014611369529813265 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 51 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764831207.9315128, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1132087.043105905, + "end_time": 1132297.989069005, + "total_evaluation_time_seconds": "210.94596310006455" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_8/boolq_2025-12-04T14-52-02.330620.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_8/boolq_2025-12-04T14-52-02.330620.json new file mode 100644 index 0000000000000000000000000000000000000000..307b61a1fdccf176cbaab8db0ed26cec2b083ca5 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_8/boolq_2025-12-04T14-52-02.330620.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.7327217125382263, + "acc_stderr,none": 0.00774005256694996 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764831014.8730035, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1131893.564729807, + "end_time": 1132036.48828004, + "total_evaluation_time_seconds": "142.92355023301207" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_8/hellaswag_2025-12-04T15-16-00.091507.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_8/hellaswag_2025-12-04T15-16-00.091507.json new file mode 100644 index 0000000000000000000000000000000000000000..5520cdb6950de8cbe4aeaf9a594b3993843fb9d6 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_8/hellaswag_2025-12-04T15-16-00.091507.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.481876120294762, + "acc_stderr,none": 0.0049865022969311935, + "acc_norm,none": 0.6820354511053575, + "acc_norm_stderr,none": 0.00464733887764219 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764831763.6470344, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1132643.845150528, + "end_time": 1133474.249558618, + "total_evaluation_time_seconds": "830.4044080900494" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_8/piqa_2025-12-04T15-01-16.294659.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_8/piqa_2025-12-04T15-01-16.294659.json new file mode 100644 index 0000000000000000000000000000000000000000..2643fc792801a1aa1e25b52219e7ca04085ea484 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_8/piqa_2025-12-04T15-01-16.294659.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.7415669205658324, + "acc_stderr,none": 0.010213971636773315, + "acc_norm,none": 0.7573449401523396, + "acc_norm_stderr,none": 0.010002002569708696 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764831629.8832, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1132510.031790193, + "end_time": 1132590.453144353, + "total_evaluation_time_seconds": "80.42135416017845" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_8/truthfulqa_mc1_2025-12-04T14-59-03.329886.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_8/truthfulqa_mc1_2025-12-04T14-59-03.329886.json new file mode 100644 index 0000000000000000000000000000000000000000..92ff6da508f637361355d77c730171ed3e6b2552 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_8/truthfulqa_mc1_2025-12-04T14-59-03.329886.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.29008567931456547, + "acc_stderr,none": 0.01588623687420952 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764831475.0147069, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1132355.245016447, + "end_time": 1132457.487958984, + "total_evaluation_time_seconds": "102.24294253694825" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_8/winogrande_2025-12-04T14-48-43.408718.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_8/winogrande_2025-12-04T14-48-43.408718.json new file mode 100644 index 0000000000000000000000000000000000000000..51b1d5147e72deb002a31bbe64f442be9f2cb9c3 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_8/winogrande_2025-12-04T14-48-43.408718.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.5927387529597474, + "acc_stderr,none": 0.013808654122417854 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764830866.090969, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1131743.885785947, + "end_time": 1131837.567074424, + "total_evaluation_time_seconds": "93.68128847680055" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_9/arc_challenge_2025-12-04T15-30-34.777559.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_9/arc_challenge_2025-12-04T15-30-34.777559.json new file mode 100644 index 0000000000000000000000000000000000000000..ba4fb3faaf9f1e13e9d72c4b8fbd426f2ff51a9d --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_9/arc_challenge_2025-12-04T15-30-34.777559.json @@ -0,0 +1,126 @@ +{ + "results": { + "arc_challenge": { + "alias": "arc_challenge", + "acc,none": 0.38054607508532423, + "acc_stderr,none": 0.014188277712349819, + "acc_norm,none": 0.41467576791808874, + "acc_norm_stderr,none": 0.014397070564409172 + } + }, + "group_subtasks": { + "arc_challenge": [] + }, + "configs": { + "arc_challenge": { + "task": "arc_challenge", + "tag": [ + "ai2_arc" + ], + "dataset_path": "allenai/ai2_arc", + "dataset_name": "ARC-Challenge", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "doc_to_text": "Question: {{question}}\nAnswer:", + "doc_to_target": "{{choices.label.index(answerKey)}}", + "unsafe_code": false, + "doc_to_choice": "{{choices.text}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 25, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "arc_challenge": 1.0 + }, + "n-shot": { + "arc_challenge": 25 + }, + "higher_is_better": { + "arc_challenge": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "arc_challenge": { + "original": 1172, + "effective": 1172 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 51 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764833268.0995288, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1134148.896430309, + "end_time": 1134348.935746871, + "total_evaluation_time_seconds": "200.0393165620044" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_9/boolq_2025-12-04T15-26-22.912502.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_9/boolq_2025-12-04T15-26-22.912502.json new file mode 100644 index 0000000000000000000000000000000000000000..5b9abb0cb2fec2d73e224d4d422e9893170eedc4 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_9/boolq_2025-12-04T15-26-22.912502.json @@ -0,0 +1,118 @@ +{ + "results": { + "boolq": { + "alias": "boolq", + "acc,none": 0.6510703363914373, + "acc_stderr,none": 0.008336340399970098 + } + }, + "group_subtasks": { + "boolq": [] + }, + "configs": { + "boolq": { + "task": "boolq", + "tag": [ + "super-glue-lm-eval-v1" + ], + "dataset_path": "super_glue", + "dataset_name": "boolq", + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "{{passage}}\nQuestion: {{question}}?\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": [ + "no", + "yes" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc" + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "passage", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "boolq": 2.0 + }, + "n-shot": { + "boolq": 0 + }, + "higher_is_better": { + "boolq": { + "acc": true + } + }, + "n-samples": { + "boolq": { + "original": 3270, + "effective": 3270 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 45 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764833081.4183092, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1133960.816319119, + "end_time": 1134097.070851929, + "total_evaluation_time_seconds": "136.25453281006776" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_9/hellaswag_2025-12-04T15-50-46.446830.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_9/hellaswag_2025-12-04T15-50-46.446830.json new file mode 100644 index 0000000000000000000000000000000000000000..431b63f6103da89184f457c40e3e5be59c3ffdac --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_9/hellaswag_2025-12-04T15-50-46.446830.json @@ -0,0 +1,127 @@ +{ + "results": { + "hellaswag": { + "alias": "hellaswag", + "acc,none": 0.42780322644891455, + "acc_stderr,none": 0.00493749019948948, + "acc_norm,none": 0.5902210714997013, + "acc_norm_stderr,none": 0.00490787714472003 + } + }, + "group_subtasks": { + "hellaswag": [] + }, + "configs": { + "hellaswag": { + "task": "hellaswag", + "tag": [ + "multiple_choice" + ], + "dataset_path": "hellaswag", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "{{query}}", + "doc_to_target": "{{label}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 10, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "hellaswag": 1.0 + }, + "n-shot": { + "hellaswag": 10 + }, + "higher_is_better": { + "hellaswag": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "hellaswag": { + "original": 10042, + "effective": 10042 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 57 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764833830.4625738, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1134712.800634361, + "end_time": 1135560.605138824, + "total_evaluation_time_seconds": "847.8045044629835" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_9/piqa_2025-12-04T15-35-48.178682.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_9/piqa_2025-12-04T15-35-48.178682.json new file mode 100644 index 0000000000000000000000000000000000000000..d7aba367d6b2f1c0a2358afdf01879418e35749c --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_9/piqa_2025-12-04T15-35-48.178682.json @@ -0,0 +1,124 @@ +{ + "results": { + "piqa": { + "alias": "piqa", + "acc,none": 0.6920565832426551, + "acc_stderr,none": 0.010770892367463683, + "acc_norm,none": 0.6871599564744287, + "acc_norm_stderr,none": 0.010817714425701078 + } + }, + "group_subtasks": { + "piqa": [] + }, + "configs": { + "piqa": { + "task": "piqa", + "dataset_path": "baber/piqa", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "Question: {{goal}}\nAnswer:", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{[sol1, sol2]}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "goal", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "piqa": 1.0 + }, + "n-shot": { + "piqa": 0 + }, + "higher_is_better": { + "piqa": { + "acc": true, + "acc_norm": true + } + }, + "n-samples": { + "piqa": { + "original": 1838, + "effective": 1838 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764833698.1640294, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1134577.699813557, + "end_time": 1134662.336979021, + "total_evaluation_time_seconds": "84.63716546399519" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_9/truthfulqa_mc1_2025-12-04T15-33-32.686607.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_9/truthfulqa_mc1_2025-12-04T15-33-32.686607.json new file mode 100644 index 0000000000000000000000000000000000000000..7b63123a13669a6545c2bc4e8bdc21ac307dc5c5 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_9/truthfulqa_mc1_2025-12-04T15-33-32.686607.json @@ -0,0 +1,116 @@ +{ + "results": { + "truthfulqa_mc1": { + "alias": "truthfulqa_mc1", + "acc,none": 0.23133414932680538, + "acc_stderr,none": 0.01476194517486267 + } + }, + "group_subtasks": { + "truthfulqa_mc1": [] + }, + "configs": { + "truthfulqa_mc1": { + "task": "truthfulqa_mc1", + "tag": [ + "truthfulqa" + ], + "dataset_path": "truthful_qa", + "dataset_name": "multiple_choice", + "validation_split": "validation", + "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "{{mc1_targets.choices}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "truthfulqa_mc1": 2.0 + }, + "n-shot": { + "truthfulqa_mc1": 0 + }, + "higher_is_better": { + "truthfulqa_mc1": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_mc1": { + "original": 817, + "effective": 817 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764833518.3954031, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1134399.849714889, + "end_time": 1134526.844926727, + "total_evaluation_time_seconds": "126.99521183804609" +} \ No newline at end of file diff --git a/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_9/winogrande_2025-12-04T15-23-16.068723.json b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_9/winogrande_2025-12-04T15-23-16.068723.json new file mode 100644 index 0000000000000000000000000000000000000000..63d9130b8c41dce708064c1c23c3e901d68422b0 --- /dev/null +++ b/lm-evaluation-harness/results3/Qwen2.5-7B-quantization-layer-mlp/mlp_9/winogrande_2025-12-04T15-23-16.068723.json @@ -0,0 +1,117 @@ +{ + "results": { + "winogrande": { + "alias": "winogrande", + "acc,none": 0.5430149960536701, + "acc_stderr,none": 0.01400038676159829 + } + }, + "group_subtasks": { + "winogrande": [] + }, + "configs": { + "winogrande": { + "task": "winogrande", + "dataset_path": "winogrande", + "dataset_name": "winogrande_xl", + "dataset_kwargs": { + "trust_remote_code": true + }, + "training_split": "train", + "validation_split": "validation", + "doc_to_text": "def doc_to_text(doc):\n answer_to_num = {\"1\": 0, \"2\": 1}\n return answer_to_num[doc[\"answer\"]]\n", + "doc_to_target": "def doc_to_target(doc):\n idx = doc[\"sentence\"].index(\"_\") + 1\n return doc[\"sentence\"][idx:].strip()\n", + "unsafe_code": false, + "doc_to_choice": "def doc_to_choice(doc):\n idx = doc[\"sentence\"].index(\"_\")\n options = [doc[\"option1\"], doc[\"option2\"]]\n return [doc[\"sentence\"][:idx] + opt for opt in options]\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 5, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "sentence", + "metadata": { + "version": 1.0, + "pretrained": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp" + } + } + }, + "versions": { + "winogrande": 1.0 + }, + "n-shot": { + "winogrande": 5 + }, + "higher_is_better": { + "winogrande": { + "acc": true + } + }, + "n-samples": { + "winogrande": { + "original": 1267, + "effective": 1267 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_num_parameters": 7615616512, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto", + "batch_sizes": [ + 64 + ], + "device": "cuda", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1764832944.9910667, + "pretty_env_info": "PyTorch version: 2.9.1+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.14 (main, Oct 21 2025, 18:31:21) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.4.143.bsk.7-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 12.4.131\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.4.0\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.4.0\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 57 bits virtual\nByte Order: Little Endian\nCPU(s): 128\nOn-line CPU(s) list: 0-127\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) Platinum 8336C CPU @ 2.30GHz\nCPU family: 6\nModel: 106\nThread(s) per core: 2\nCore(s) per socket: 32\nSocket(s): 2\nStepping: 6\nCPU(s) scaling MHz: 86%\nCPU max MHz: 3500.0000\nCPU min MHz: 800.0000\nBogoMIPS: 4600.00\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities\nVirtualization: VT-x\nL1d cache: 3 MiB (64 instances)\nL1i cache: 2 MiB (64 instances)\nL2 cache: 80 MiB (64 instances)\nL3 cache: 108 MiB (2 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-31,64-95\nNUMA node1 CPU(s): 32-63,96-127\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] mypy==1.19.0\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==1.26.4\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] open_clip_torch==3.2.0\n[pip3] torch==2.9.1\n[pip3] torchvision==0.24.1\n[pip3] triton==3.5.1\n[conda] numpy 1.26.4 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.8.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.8.90 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.10.2.21 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.3.83 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.9.90 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.3.90 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.8.93 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.7.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.27.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.8.93 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.8.90 pypi_0 pypi\n[conda] open-clip-torch 3.2.0 pypi_0 pypi\n[conda] torch 2.9.1 pypi_0 pypi\n[conda] torchvision 0.24.1 pypi_0 pypi\n[conda] triton 3.5.1 pypi_0 pypi", + "transformers_version": "4.57.3", + "lm_eval_version": "0.4.8", + "upper_git_hash": "3761bde4a46223e738034eac9a2e68a7b5997d5e", + "tokenizer_pad_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_eos_token": [ + "<|endoftext|>", + "151643" + ], + "tokenizer_bos_token": [ + null, + "None" + ], + "eot_token_id": 151643, + "max_length": 131072, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B-quantization-layer-mlp", + "model_name_sanitized": "__mnt__bn__life-mllm__users__cxr__quantization__models__Qwen2.5-7B-quantization-layer-mlp", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 1133825.877045797, + "end_time": 1133910.226947778, + "total_evaluation_time_seconds": "84.34990198095329" +} \ No newline at end of file diff --git a/lm-evaluation-harness/run_scripts/eval.sh b/lm-evaluation-harness/run_scripts/eval.sh new file mode 100644 index 0000000000000000000000000000000000000000..6a0c509363023071330dac48494ed6849501fe99 --- /dev/null +++ b/lm-evaluation-harness/run_scripts/eval.sh @@ -0,0 +1,15 @@ +#!/bin/bash +configure_id=$1 +model=$2 +# output_dir=$3 +task=$3 +dir=$4 +# task=hellaswag,rte,mmlu +echo "${configure_id} ${task}" +lm_eval --model hf \ + --model_args pretrained=${model} \ + --tasks $task \ + --device cuda:0 \ + --batch_size 16 \ + --output_path results/${dir}/${configure_id}.json + diff --git a/lm-evaluation-harness/run_scripts/eval_all.sh b/lm-evaluation-harness/run_scripts/eval_all.sh new file mode 100644 index 0000000000000000000000000000000000000000..92f32c00bc092e1432adad8db6ebe9390155da32 --- /dev/null +++ b/lm-evaluation-harness/run_scripts/eval_all.sh @@ -0,0 +1,2 @@ +# bash eval_base_qwen2_5_7b.sh +nohup bash run_scripts/eval_base_qwen2_5_32b.sh > eval_base_qwen2_5_32b.log 2>&1 & \ No newline at end of file diff --git a/lm-evaluation-harness/run_scripts/eval_base.sh b/lm-evaluation-harness/run_scripts/eval_base.sh new file mode 100644 index 0000000000000000000000000000000000000000..9b3de4e8db56b743cc92a05d8cf0a5e28bfe1dba --- /dev/null +++ b/lm-evaluation-harness/run_scripts/eval_base.sh @@ -0,0 +1,27 @@ +#!/bin/bash +export HTTP_PROXY=http://sys-proxy-rd-relay.byted.org:8118 +export http_proxy=http://sys-proxy-rd-relay.byted.org:8118 +export https_proxy=http://sys-proxy-rd-relay.byted.org:8118 +export no_proxy="$no_proxy,.byteintl.net" +export HF_ENDPOINT=https://hf-mirror.com + +model=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen2.5-7B +# output_dir=$3 +task=hellaswag +# task=hellaswag,rte,mmlu +lm_eval --model hf \ + --model_args pretrained=${model} \ + --tasks hellaswag \ + --device cuda:0 \ + --batch_size 64 \ + --num_fewshot 10 \ + --output_path results2/base/base_${model}.json + +# lm_eval --model hf \ +# --model_args pretrained=${model} \ +# --tasks arc_challenge \ +# --device cuda:1 \ +# --batch_size 16 \ +# --num_fewshot 25 \ +# --output_path results2/base/base_${model}.json + diff --git a/lm-evaluation-harness/run_scripts/eval_base_baseline.sh b/lm-evaluation-harness/run_scripts/eval_base_baseline.sh new file mode 100644 index 0000000000000000000000000000000000000000..03465234058f2170cea1e210c231ff3042f55c5c --- /dev/null +++ b/lm-evaluation-harness/run_scripts/eval_base_baseline.sh @@ -0,0 +1,58 @@ +# export HTTP_PROXY=http://sys-proxy-rd-relay.byted.org:8118 +# export http_proxy=http://sys-proxy-rd-relay.byted.org:8118 +# export https_proxy=http://sys-proxy-rd-relay.byted.org:8118 +# export no_proxy="$no_proxy,.byteintl.net" +# export HF_ENDPOINT=https://hf-mirror.com + +model=$1 +output_dir=$2 +model_name=$(basename "$model") + +output_dir="results2/${model_name}/${output_dir}" + +# model=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen/Qwen2.5-14B +# model_name=$(basename "$model") + +mkdir -p "$output_dir" # -p 确保父目录存在，无则创建 + +# 3. 定义「任务名→num_fewshot」映射（核心：新增/删除任务只改这里） +declare -A task_fewshot=( + ["winogrande"]="5" + ["truthfulqa_mc1"]="0" + ["arc_challenge"]="25" + ["hellaswag"]="10" + # ["mmlu"]="5" + # ["gsm8k"]="4" + ["piqa"]="0" + ["boolq"]="0" +) + +# 4. 循环执行所有任务（重复逻辑一次性写死） +for task in "${!task_fewshot[@]}"; do + num_fewshot=${task_fewshot[$task]} + output_path="${output_dir}/${task}.json" + + # 打印进度信息 + echo -e "\n==================================================" + echo "开始评估：任务=$task | 少样本数=$num_fewshot | 模型=$model_name" + echo "输出路径：$output_path" + echo "==================================================" + + # 核心评估命令（仅替换动态变量） + CUDA_VISIBLE_DEVICES=0,1,2,3 \ + accelerate launch --num_processes=4 --main_process_port 1472 \ + lm_eval --model hf \ + --model_args "pretrained=${model}" \ + --tasks "$task" \ + --device cuda \ + --batch_size auto \ + --num_fewshot "$num_fewshot" \ + --output_path "$output_path" + + # 检查命令是否执行成功 + if [ $? -eq 0 ]; then + echo "任务 $task 评估完成！" + else + echo "警告：任务 $task 执行失败！" + fi +done diff --git a/lm-evaluation-harness/run_scripts/eval_base_fg.sh b/lm-evaluation-harness/run_scripts/eval_base_fg.sh new file mode 100644 index 0000000000000000000000000000000000000000..2e77418a0b64ffb637a0fa17cac7ff6516e4b2b4 --- /dev/null +++ b/lm-evaluation-harness/run_scripts/eval_base_fg.sh @@ -0,0 +1,58 @@ +# export HTTP_PROXY=http://sys-proxy-rd-relay.byted.org:8118 +# export http_proxy=http://sys-proxy-rd-relay.byted.org:8118 +# export https_proxy=http://sys-proxy-rd-relay.byted.org:8118 +# export no_proxy="$no_proxy,.byteintl.net" +# export HF_ENDPOINT=https://hf-mirror.com + +model=$1 +output_dir=$2 +model_name=$(basename "$model") + +output_dir="results2/${model_name}/${output_dir}" + +# model=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen/Qwen2.5-14B +# model_name=$(basename "$model") + +mkdir -p "$output_dir" # -p 确保父目录存在，无则创建 + +# 3. 定义「任务名→num_fewshot」映射（核心：新增/删除任务只改这里） +declare -A task_fewshot=( + ["winogrande"]="5" + ["truthfulqa_mc1"]="0" + ["arc_challenge"]="25" + ["hellaswag"]="10" + # ["mmlu"]="5" + # ["gsm8k"]="4" + ["piqa"]="0" + ["boolq"]="0" +) + +# 4. 循环执行所有任务（重复逻辑一次性写死） +for task in "${!task_fewshot[@]}"; do + num_fewshot=${task_fewshot[$task]} + output_path="${output_dir}/${task}.json" + + # 打印进度信息 + echo -e "\n==================================================" + echo "开始评估：任务=$task | 少样本数=$num_fewshot | 模型=$model_name" + echo "输出路径：$output_path" + echo "==================================================" + + # 核心评估命令（仅替换动态变量） + CUDA_VISIBLE_DEVICES=6,7 \ + accelerate launch --num_processes=2 --main_process_port 1476 \ + lm_eval --model hf \ + --model_args "pretrained=${model}" \ + --tasks "$task" \ + --device cuda \ + --batch_size auto \ + --num_fewshot "$num_fewshot" \ + --output_path "$output_path" + + # 检查命令是否执行成功 + if [ $? -eq 0 ]; then + echo "任务 $task 评估完成！" + else + echo "警告：任务 $task 执行失败！" + fi +done diff --git a/lm-evaluation-harness/run_scripts/eval_base_llama3-8b.sh b/lm-evaluation-harness/run_scripts/eval_base_llama3-8b.sh new file mode 100644 index 0000000000000000000000000000000000000000..08f18da0705f7fb53ce9c5a70ad03e8183834395 --- /dev/null +++ b/lm-evaluation-harness/run_scripts/eval_base_llama3-8b.sh @@ -0,0 +1,51 @@ +export HTTP_PROXY=http://sys-proxy-rd-relay.byted.org:8118 +export http_proxy=http://sys-proxy-rd-relay.byted.org:8118 +export https_proxy=http://sys-proxy-rd-relay.byted.org:8118 +export no_proxy="$no_proxy,.byteintl.net" +export HF_ENDPOINT=https://hf-mirror.com + +model=/mnt/bn/life-mllm/users/cxr/quantization/models/meta-llama/Llama-3.1-8B +# output_dir=$3 +task=winogrande,arc_challenge,hellaswag,mmlu,truthfulqa_mc1,triviaqa +model_name=$(basename "$model") + +output_dir="results2/${model_name}" +mkdir -p "$output_dir" # -p 确保父目录存在，无则创建 + +# 3. 定义「任务名→num_fewshot」映射（核心：新增/删除任务只改这里） +declare -A task_fewshot=( + ["winogrande"]="5" + ["truthfulqa_mc1"]="0" + ["arc_challenge"]="25" + ["mmlu"]="5" + ["triviaqa"]="0" + ["hellaswag"]="10" +) + +# 4. 循环执行所有任务（重复逻辑一次性写死） +for task in "${!task_fewshot[@]}"; do + num_fewshot=${task_fewshot[$task]} + output_path="${output_dir}/base_${task}.json" + + # 打印进度信息 + echo -e "\n==================================================" + echo "开始评估：任务=$task | 少样本数=$num_fewshot | 模型=$model_name" + echo "输出路径：$output_path" + echo "==================================================" + + # 核心评估命令（仅替换动态变量） + lm_eval --model hf \ + --model_args "pretrained=${model}" \ + --tasks "$task" \ + --device cuda:2 \ + --batch_size auto \ + --num_fewshot "$num_fewshot" \ + --output_path "$output_path" + + # 检查命令是否执行成功 + if [ $? -eq 0 ]; then + echo "任务 $task 评估完成！" + else + echo "警告：任务 $task 执行失败！" + fi +done diff --git a/lm-evaluation-harness/run_scripts/eval_base_qwen2_5_14b.sh b/lm-evaluation-harness/run_scripts/eval_base_qwen2_5_14b.sh new file mode 100644 index 0000000000000000000000000000000000000000..dc8eb3daef45c48deb14c7745fc5365ed3410e30 --- /dev/null +++ b/lm-evaluation-harness/run_scripts/eval_base_qwen2_5_14b.sh @@ -0,0 +1,58 @@ +# export HTTP_PROXY=http://sys-proxy-rd-relay.byted.org:8118 +# export http_proxy=http://sys-proxy-rd-relay.byted.org:8118 +# export https_proxy=http://sys-proxy-rd-relay.byted.org:8118 +# export no_proxy="$no_proxy,.byteintl.net" +# export HF_ENDPOINT=https://hf-mirror.com + +model=$1 +output_dir=$2 +model_name=$(basename "$model") + +output_dir="results2/${model_name}/${output_dir}" + +# model=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen/Qwen2.5-14B +# model_name=$(basename "$model") + +mkdir -p "$output_dir" # -p 确保父目录存在，无则创建 + +# 3. 定义「任务名→num_fewshot」映射（核心：新增/删除任务只改这里） +declare -A task_fewshot=( + ["winogrande"]="5" + ["truthfulqa_mc1"]="0" + ["arc_challenge"]="25" + ["hellaswag"]="10" + # ["mmlu"]="5" + # ["triviaqa"]="0" + ["piqa"]="0" + ["boolq"]="0" +) + +# 4. 循环执行所有任务（重复逻辑一次性写死） +for task in "${!task_fewshot[@]}"; do + num_fewshot=${task_fewshot[$task]} + output_path="${output_dir}/${task}.json" + + # 打印进度信息 + echo -e "\n==================================================" + echo "开始评估：任务=$task | 少样本数=$num_fewshot | 模型=$model_name" + echo "输出路径：$output_path" + echo "==================================================" + + # 核心评估命令（仅替换动态变量） + CUDA_VISIBLE_DEVICES=4,5 \ + accelerate launch --num_processes=2 --main_process_port 1494 \ + lm_eval --model hf \ + --model_args "pretrained=${model}" \ + --tasks "$task" \ + --device cuda \ + --batch_size auto \ + --num_fewshot "$num_fewshot" \ + --output_path "$output_path" + + # 检查命令是否执行成功 + if [ $? -eq 0 ]; then + echo "任务 $task 评估完成！" + else + echo "警告：任务 $task 执行失败！" + fi +done diff --git a/lm-evaluation-harness/run_scripts/eval_base_qwen2_5_32b.sh b/lm-evaluation-harness/run_scripts/eval_base_qwen2_5_32b.sh new file mode 100644 index 0000000000000000000000000000000000000000..61308780c2cb91271fa7306d09cf3710cd7a850f --- /dev/null +++ b/lm-evaluation-harness/run_scripts/eval_base_qwen2_5_32b.sh @@ -0,0 +1,51 @@ +export HTTP_PROXY=http://sys-proxy-rd-relay.byted.org:8118 +export http_proxy=http://sys-proxy-rd-relay.byted.org:8118 +export https_proxy=http://sys-proxy-rd-relay.byted.org:8118 +export no_proxy="$no_proxy,.byteintl.net" +export HF_ENDPOINT=https://hf-mirror.com + +model=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen/Qwen2.5-32B +model_name=$(basename "$model") + +output_dir="results2/${model_name}" +mkdir -p "$output_dir" # -p 确保父目录存在，无则创建 + +# 3. 定义「任务名→num_fewshot」映射（核心：新增/删除任务只改这里） +declare -A task_fewshot=( + ["winogrande"]="5" + ["truthfulqa_mc1"]="0" + ["arc_challenge"]="25" + ["hellaswag"]="10" + ["mmlu"]="5" + ["triviaqa"]="0" +) + +# 4. 循环执行所有任务（重复逻辑一次性写死） +for task in "${!task_fewshot[@]}"; do + num_fewshot=${task_fewshot[$task]} + output_path="${output_dir}/base_${task}.json" + + # 打印进度信息 + echo -e "\n==================================================" + echo "开始评估：任务=$task | 少样本数=$num_fewshot | 模型=$model_name" + echo "输出路径：$output_path" + echo "==================================================" + + # 核心评估命令（仅替换动态变量） + CUDA_VISIBLE_DEVICES=6,7 \ + accelerate launch --num_processes=2 --main_process_port 1435 \ + lm_eval --model hf \ + --model_args "pretrained=${model}" \ + --tasks "$task" \ + --device cuda \ + --batch_size auto \ + --num_fewshot "$num_fewshot" \ + --output_path "$output_path" + + # 检查命令是否执行成功 + if [ $? -eq 0 ]; then + echo "任务 $task 评估完成！" + else + echo "警告：任务 $task 执行失败！" + fi +done diff --git a/lm-evaluation-harness/run_scripts/eval_base_qwen2_5_7b.sh b/lm-evaluation-harness/run_scripts/eval_base_qwen2_5_7b.sh new file mode 100644 index 0000000000000000000000000000000000000000..45b0e8256151b685701ce082388a601166efde1f --- /dev/null +++ b/lm-evaluation-harness/run_scripts/eval_base_qwen2_5_7b.sh @@ -0,0 +1,52 @@ +# export HTTP_PROXY=http://sys-proxy-rd-relay.byted.org:8118 +# export http_proxy=http://sys-proxy-rd-relay.byted.org:8118 +# export https_proxy=http://sys-proxy-rd-relay.byted.org:8118 +# export no_proxy="$no_proxy,.byteintl.net" +# export HF_ENDPOINT=https://hf-mirror.com + +model=$1 +output_dir=$2 +cuda_id=$3 +model_name=$(basename "$model") + +output_dir="results2/${model_name}/${output_dir}" +mkdir -p "$output_dir" # -p 确保父目录存在，无则创建 +# 3. 定义「任务名→num_fewshot」映射（核心：新增/删除任务只改这里） +declare -A task_fewshot=( + ["winogrande"]="5" + ["truthfulqa_mc1"]="0" + ["arc_challenge"]="25" + ["hellaswag"]="10" + # ["mmlu"]="5" + # ["triviaqa"]="0" + ["piqa"]="0" + ["boolq"]="0" +) + +# 4. 循环执行所有任务（重复逻辑一次性写死） +for task in "${!task_fewshot[@]}"; do + num_fewshot=${task_fewshot[$task]} + output_path="${output_dir}/${task}.json" + + # 打印进度信息 + echo -e "\n==================================================" + echo "开始评估：任务=$task | 少样本数=$num_fewshot | 模型=$model_name" + echo "输出路径：$output_path" + echo "==================================================" + + # 核心评估命令（仅替换动态变量） + lm_eval --model hf \ + --model_args "pretrained=${model}" \ + --tasks "$task" \ + --device cuda:${cuda_id} \ + --batch_size auto \ + --num_fewshot "$num_fewshot" \ + --output_path "$output_path" + + # 检查命令是否执行成功 + if [ $? -eq 0 ]; then + echo "任务 $task 评估完成！" + else + echo "警告：任务 $task 执行失败！" + fi +done diff --git a/lm-evaluation-harness/run_scripts/eval_low.sh b/lm-evaluation-harness/run_scripts/eval_low.sh new file mode 100644 index 0000000000000000000000000000000000000000..1f8e8822f5fa9129d4133a7220bfd9ef140c750a --- /dev/null +++ b/lm-evaluation-harness/run_scripts/eval_low.sh @@ -0,0 +1,29 @@ +#!/bin/bash +configure_id=$1 +model=$2 +output_dir=$3 +# task=piqa,winogrande,arc_easy,arc_challenge,hellaswag,boolq +task=hellaswag,rte,mmlu +echo "${configure_id} ${task}" +lm_eval --model hf \ + --model_args pretrained=${model} \ + --tasks $task \ + --device cuda:0 \ + --batch_size 32 \ + --output_path results/${output_dir}/Llama-2-7b-hf-${configure_id}.json + +# tasks=(piqa winogrande hellaswag arc_easy arc_challenge wikitext) +# models=('Llama-2-7b-hf') + +# for task in "${tasks[@]}"; do +# mkdir -p results/${task} +# for model in "${models[@]}"; do +# echo "${model} ${task}" +# lm_eval --model hf \ +# --model_args pretrained=../models/${model} \ +# --tasks $task \ +# --device cuda:0 \ +# --batch_size 32 \ +# --output_path results/${model}/${task}.json +# done +# done diff --git a/lm-evaluation-harness/run_scripts/eval_test.sh b/lm-evaluation-harness/run_scripts/eval_test.sh new file mode 100644 index 0000000000000000000000000000000000000000..b5b0c7bf3c33d8641fdaa0b917b5966e76fb3d31 --- /dev/null +++ b/lm-evaluation-harness/run_scripts/eval_test.sh @@ -0,0 +1,59 @@ +# export HTTP_PROXY=http://sys-proxy-rd-relay.byted.org:8118 +# export http_proxy=http://sys-proxy-rd-relay.byted.org:8118 +# export https_proxy=http://sys-proxy-rd-relay.byted.org:8118 +# export no_proxy="$no_proxy,.byteintl.net" +# export HF_ENDPOINT=https://hf-mirror.com + +model=$1 +output_dir=$2 +cuda_id=$3 +model_name=$(basename "$model") + +output_dir="results3/${model_name}/${output_dir}" + +# model=/mnt/bn/life-mllm/users/cxr/quantization/models/Qwen/Qwen2.5-14B +# model_name=$(basename "$model") + +mkdir -p "$output_dir" # -p 确保父目录存在，无则创建 + +# 3. 定义「任务名→num_fewshot」映射（核心：新增/删除任务只改这里） +declare -A task_fewshot=( + ["winogrande"]="5" + ["truthfulqa_mc1"]="0" + ["arc_challenge"]="25" + ["hellaswag"]="10" + # ["mmlu"]="5" + # ["triviaqa"]="0" + ["piqa"]="0" + ["boolq"]="0" +) + +# 4. 循环执行所有任务（重复逻辑一次性写死） +for task in "${!task_fewshot[@]}"; do + num_fewshot=${task_fewshot[$task]} + output_path="${output_dir}/${task}.json" + + # 打印进度信息 + echo -e "\n==================================================" + echo "开始评估：任务=$task | 少样本数=$num_fewshot | 模型=$model_name" + echo "输出路径：$output_path" + echo "==================================================" + + # 核心评估命令（仅替换动态变量） + CUDA_VISIBLE_DEVICES=2,3,6,7 \ + accelerate launch --num_processes=4 --main_process_port 1445 \ + lm_eval --model hf \ + --model_args "pretrained=${model}" \ + --tasks "$task" \ + --device cuda \ + --batch_size auto \ + --num_fewshot "$num_fewshot" \ + --output_path "$output_path" + + # 检查命令是否执行成功 + if [ $? -eq 0 ]; then + echo "任务 $task 评估完成！" + else + echo "警告：任务 $task 执行失败！" + fi +done diff --git a/lm-evaluation-harness/scripts/__init__.py b/lm-evaluation-harness/scripts/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lm-evaluation-harness/scripts/build_benchmark.py b/lm-evaluation-harness/scripts/build_benchmark.py new file mode 100644 index 0000000000000000000000000000000000000000..3851cdb9fbd6e6035a70d5414084ee38b1689ef0 --- /dev/null +++ b/lm-evaluation-harness/scripts/build_benchmark.py @@ -0,0 +1,63 @@ +import argparse +import logging +import os + +import yaml +from promptsource.templates import DatasetTemplates +from tqdm import tqdm + + +# from lm_eval.api.registry import ALL_TASKS +eval_logger = logging.getLogger(__name__) + + +# from lm_eval.tasks import include_task_folder + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--benchmark_name", required=True) + parser.add_argument("--benchmark_path", required=True) + parser.add_argument("--task_save_path", default="lm_eval/tasks/") + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() + + with open(args.benchmark_path, encoding="utf-8") as file: + TASK_LIST = yaml.full_load(file) + for task in tqdm(TASK_LIST): + eval_logger.info(f"Processing {task}") + + dataset_name = task["dataset_path"] + if "dataset_name" in task: + subset_name = task["dataset_name"] + file_subdir = f"{dataset_name}/{subset_name}" + else: + subset_name = None + file_subdir = f"{dataset_name}" + + file_path = os.path.join(args.task_save_path, file_subdir, "promptsource/") + + os.makedirs(file_path, exist_ok=True) + + if subset_name is None: + prompts = DatasetTemplates(dataset_name=dataset_name) + else: + prompts = DatasetTemplates( + dataset_name=dataset_name, subset_name=subset_name + ) + + for idx, prompt_name in enumerate(prompts.all_template_names): + full_file_name = f"promptsource_{idx}.yaml" + config_dict = { + "group": args.benchmark_name, + "include": "promptsource_template.yaml", + "use_prompts": f"promptsource:{prompt_name}", + } + + file_save_path = os.path.join(file_path, full_file_name) + eval_logger.info(f"Save to {file_save_path}") + with open(file_save_path, "w", encoding="utf-8") as yaml_file: + yaml.dump(config_dict, yaml_file) diff --git a/lm-evaluation-harness/scripts/clean_training_data/README.md b/lm-evaluation-harness/scripts/clean_training_data/README.md new file mode 100644 index 0000000000000000000000000000000000000000..f06a4996b165a8dc4c174975687d0a32d6051eaf --- /dev/null +++ b/lm-evaluation-harness/scripts/clean_training_data/README.md @@ -0,0 +1,41 @@ +# Clean Training Data + +janitor.py contains a script to remove benchmark data contamination from training data sets. +It uses the approach described in the [GPT-3 paper](https://arxiv.org/abs/2005.14165). + +## Algorithm + +1) Collects all contamination text files that are to be removed from training data +2) Filters training data by finding `N`gram matches between the training data + and any contamination + 1) `N`grams ignore case and punctuation and are split on whitespace. + 2) Matching `N`gram substrings are removed, as is a `window_to_remove` character window around + the match, splitting the training data into chunks + 3) Any chunks less than `minimum_slice_length` are removed + 4) Training data sets split into more than `too_dirty_cutoff` are considered + completely contaminated and removed + +OpenAI used: + +```text +ngram_n = 13 +window_to_remove = 200 +minimum_slice_length = 200 +too_dirty_cutoff = 10 +``` + +## Compiling + +Janitor can be used as a pure python program, but it is much faster if the ngram +code is run in C++. To compile the C++ code, run + +```bash +pip install pybind11 +c++ -O3 -Wall -shared -std=c++11 -fPIC $(python3 -m pybind11 --includes) janitor_util.cpp -o janitor_util$(python3-config --extension-suffix) +``` + +MacOS users: If your compiler isn't linked to Python, you may need to add to the above `-undefined dynamic_lookup`. \ +Linux users: If your compiler isn't linked to Python, you may need to follow these steps: + +1. Rename the compiled code file to `janitor_util.so`. +2. Before running `import Janitor` in your code, add `sys.path.append("your/relative/path/to/janitor_util.so")` so that Python knows the location of `janitor_util.so`. diff --git a/lm-evaluation-harness/scripts/clean_training_data/__init__.py b/lm-evaluation-harness/scripts/clean_training_data/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lm-evaluation-harness/scripts/clean_training_data/compress_and_package.py b/lm-evaluation-harness/scripts/clean_training_data/compress_and_package.py new file mode 100644 index 0000000000000000000000000000000000000000..d4af5ba5f3d5e16a485984ced2324951e56ad829 --- /dev/null +++ b/lm-evaluation-harness/scripts/clean_training_data/compress_and_package.py @@ -0,0 +1,73 @@ +import argparse +import glob +import logging +import os +import shutil +import subprocess + +from tqdm import tqdm +from tqdm_multiprocess import TqdmMultiProcessPool +from tqdm_multiprocess.logger import setup_logger_tqdm + + +logger = logging.getLogger(__name__) + + +def process_task( + working_directory, output_directory, bucket_file_path, tqdm_func, global_tqdm +): + command = f"zstd {bucket_file_path}" + logger.info(command) + subprocess.call(command, shell=True) + + compressed_file = bucket_file_path + ".zst" + if output_directory: + shutil.move(compressed_file, output_directory) + + os.remove(bucket_file_path) + global_tqdm.update() + + +def compress_and_move(working_directory, output_directory, process_count): + os.makedirs(output_directory, exist_ok=True) + original_info_file_path = os.path.join(working_directory, "info.json") + assert os.path.exists(original_info_file_path) + + tasks = [] + bucket_file_paths = glob.glob( + os.path.join(working_directory, "output", "*.bkt.txt.sorted") + ) + for bucket_file_path in bucket_file_paths: + task = (process_task, (working_directory, output_directory, bucket_file_path)) + tasks.append(task) + + pool = TqdmMultiProcessPool(process_count) + + def on_done(_): + return None + + def on_error(_): + return None + + global_progress = tqdm( + total=len(bucket_file_paths), dynamic_ncols=True, unit="file" + ) + _ = pool.map(global_progress, tasks, on_error, on_done) + + shutil.copy(original_info_file_path, os.path.join(output_directory, "info.json")) + + +parser = argparse.ArgumentParser(description="sort 13gram buckets") +parser.add_argument("-dir", "--working_directory", required=True) +parser.add_argument("-output", "--output_directory", required=True) +parser.add_argument("-procs", "--process_count", type=int, default=8) + +if __name__ == "__main__": + version = 1.00 + print(f"Running version {version}") + + logfile_path = "compress_and_package.log" + setup_logger_tqdm(logfile_path) + + args = parser.parse_args() + compress_and_move(args.working_directory, args.output_directory, args.process_count) diff --git a/lm-evaluation-harness/scripts/clean_training_data/generate_13_grams.py b/lm-evaluation-harness/scripts/clean_training_data/generate_13_grams.py new file mode 100644 index 0000000000000000000000000000000000000000..530c35ecd7785dbb9f97c5fcecc7e1ce776cc5b0 --- /dev/null +++ b/lm-evaluation-harness/scripts/clean_training_data/generate_13_grams.py @@ -0,0 +1,215 @@ +""" +Outputs all 13-grams found in The Pile. + +Loops through all documents and uses the logic found in janitor.py to extract 13-grams. +We bucket each 13-gram by hash into separate file buckets to allow easy parallel processing in the +next stage. We also include the current pile document_id with each ngram instance to allow the +filtering to exclude 13-grams that match more then 10 unique documents (done further down the pipeline). + +We didn't use lm_dataformat to output as it increases time 4x (slow jsonify) and makes +resuming hard (and we had the storage). + +Arguments +--------- +--working_directory (-dir) + Directory containing the pile distribution. An "output" subdirectory will be created underneath + to store the bucketed 13-grams, checkpoint and done files. Default: current directory +--n_value (-n) + n value in n-gram, added for later use if ever needed. Default: 13 +--bucket_count (-buckets) + Number of file buckets to use when generating 13grams. Default: 500 +""" + +import argparse +import glob +import json +import logging +import os +import pickle +import signal +import sys +from pathlib import Path +from signal import SIGINT + +from tqdm import tqdm +from tqdm_multiprocess.logger import setup_logger_tqdm + +from lm_eval.decontamination.archiver import Reader, TextArchive +from lm_eval.decontamination.janitor import Janitor, word_ngrams + + +logger = logging.getLogger(__name__) + +terminate = False + + +def handler(signal_received, frame): + global terminate + terminate = True + + +def yield_pile(start_offsets=None, checkpoint_offset=None): + directory = "pile" + + if not os.path.exists(directory): + print( + "We expect the pile archives to be in the 'pile' directory, but this was not found." + ) + raise FileNotFoundError("Pile directory not found.") + + files = list(sorted(glob.glob(os.path.join(directory, "*.jsonl.zst*")))) + + pile_global_offset = 0 + start_file = 0 + if checkpoint_offset: + for file_i, start_offset in enumerate(start_offsets): + if start_offset > checkpoint_offset: + break + + start_file = file_i + pile_global_offset = start_offset + + for file_i, file in enumerate(files): + if file_i < start_file: + logger.info(f"Skipping file {file}") + continue + logger.info(f"Reading from pile file: {file}") + reader = Reader() + for document in reader.read(file): + yield (pile_global_offset, document) + pile_global_offset += 1 + + +# Hash buckets > disk backed files. Supports file position checkpointing and resuming +# Allows you to write continuously and checkpoint intermittently. If a failure occurs +# the buckets are simply truncated at your last checkpoint. +class Buckets: + def __init__(self, directory, num_buckets): + self.bucket_files = [ + os.path.join(directory, f"ngrams_{i}.bkt.txt") for i in range(num_buckets) + ] + self.buckets = list(map(TextArchive, self.bucket_files)) + self.checkpoint_file = os.path.join(directory, "bucket_offsets.ckpt") + + if os.path.exists(self.checkpoint_file): + self.bucket_offsets = pickle.load(open(self.checkpoint_file, "rb")) + else: + self.bucket_offsets = [0 for i in range(len(self.buckets))] + + for i, offset in enumerate(self.bucket_offsets): + bucket = self.buckets[i] + bucket.fh.seek(offset) + bucket.fh.truncate() + + def add_data(self, key, value): + i = hash(key) % len(self.buckets) + bucket = self.buckets[i] + bucket.add_data(value) + + def save_checkpoint(self): + for bucket in self.buckets: + bucket.fh.flush() + + bucket_offsets = [bucket.fh.tell() for bucket in self.buckets] + pickle.dump(bucket_offsets, open(self.checkpoint_file, "wb")) + + def close_buckets(self): + for bucket in self.buckets: + bucket.commit() + + +def do_ngrams_in_buckets(n_value, working_directory, bucket_count): + pile_statistics = json.load(open("pile_statistics.json", "r", encoding="utf-8")) + pile_document_count = pile_statistics["Document Count"] + start_offsets = pile_statistics["File Start Offsets"] + + output_directory = os.path.join(working_directory, "output") + os.makedirs(output_directory, exist_ok=True) + + logger.info(f"Generating {n_value}-grams and bucketing.") + + # Done file + done_file = os.path.join(output_directory, "ngram_buckets.done") + if os.path.exists(done_file): + logger.info("ngrams already generated and bucketed, skipping") + return + + # Checkpoint + checkpoint_file = os.path.join(working_directory, "pile_offset.ckpt") + if os.path.exists(checkpoint_file): + checkpoint_offset = pickle.load(open(checkpoint_file, "rb")) + iterate = True + else: + checkpoint_offset = 0 + iterate = False + + logger.info(f"Starting at pile document index {checkpoint_offset}") + buckets = Buckets(output_directory, bucket_count) + + janitor = Janitor() + batch_size = 1000 + batch_counter = 0 + + with tqdm(total=checkpoint_offset, dynamic_ncols=True, unit="docs") as progress: + for offset, document in yield_pile(start_offsets, checkpoint_offset): + if iterate: + logger.info(f"Iterating to offset {checkpoint_offset} from {offset}") + progress.update(offset) + iterate = False + + if offset < checkpoint_offset: + progress.update() + + if terminate: + return + continue + + if offset == checkpoint_offset: + progress.reset(total=pile_document_count) + progress.update(checkpoint_offset) + + # Save checkpoint every "batch_size", only allow terminate after checkpoint + if batch_counter == batch_size: + progress.update(batch_size) + batch_counter = 0 + buckets.save_checkpoint() + pickle.dump(offset, open(checkpoint_file, "wb")) + if terminate: + buckets.close_buckets() + return + + ngrams = word_ngrams(janitor.normalize_string(document), n_value) + for ngram in ngrams: + buckets.add_data(ngram, f"{ngram} {offset}") + + batch_counter += 1 + + buckets.close_buckets() + Path(done_file).touch() + + +parser = argparse.ArgumentParser(description="Generate 13 grams from Pile.") +parser.add_argument("-dir", "--working_directory", default="") +parser.add_argument("-n", "--n_value", type=int, default=13) +parser.add_argument("-buckets", "--bucket_count", type=int, default=500) + +if __name__ == "__main__": + version = 1.00 + print(f"Running version {version}") + + if "PYTHONHASHSEED" not in os.environ or os.environ["PYTHONHASHSEED"] != "0": + print("Please run 'export PYTHONHASHSEED=0' before running generate.") + sys.exit() + + # Handle sigint (ctrl-c) cleanly + previous_signal_int = signal.signal(SIGINT, handler) + + logfile_path = "ngrams.log" + setup_logger_tqdm(logfile_path) + + args = parser.parse_args() + do_ngrams_in_buckets(args.n_value, args.working_directory, args.bucket_count) + + info_dict = {"title": "dataset ngrams", "ngram_size": 13} + info_dict_path = os.path.join(args.working_directory, "info.json") + json.dump(info_dict, open(info_dict_path, "w", encoding="utf-8")) diff --git a/lm-evaluation-harness/scripts/clean_training_data/investigate_pile.py b/lm-evaluation-harness/scripts/clean_training_data/investigate_pile.py new file mode 100644 index 0000000000000000000000000000000000000000..681b591ced535dbb884fb65f58a0c9042c35b0ac --- /dev/null +++ b/lm-evaluation-harness/scripts/clean_training_data/investigate_pile.py @@ -0,0 +1,95 @@ +import glob +import json +import os +from functools import reduce + +import tqdm +from tqdm_multiprocess import TqdmMultiProcessPool + +from lm_eval.decontamination.archiver import Reader + + +def get_file_stats(file_path, tqdm_func, global_tqdm): + reader = Reader() + total_documents = 0 + total_size = 0 + update_frequency = 10000 + current_file_position = 0 + + with tqdm_func( + total=os.path.getsize(file_path), dynamic_ncols=True, unit="byte", unit_scale=1 + ) as progress: + for document in reader.read(file_path, get_meta=True): + total_size += len(document) + total_documents += 1 + + if total_documents % update_frequency == 0: + new_file_pos = reader.fh.tell() + bytes_read = new_file_pos - current_file_position + current_file_position = new_file_pos + progress.update(bytes_read) + global_tqdm.update(bytes_read) + + return (total_documents, total_size) + + +def get_files(): + directory = "pile" + files = list(sorted(glob.glob(os.path.join(directory, "*.jsonl.zst*")))) + print(files) + return files + + +def get_stats(): + files = get_files() + total_size_bytes = sum(map(lambda x: os.path.getsize(x), files)) + + pool = TqdmMultiProcessPool(4) + global_tqdm = tqdm.tqdm( + total=total_size_bytes, dynamic_ncols=True, unit="byte", unit_scale=1 + ) + + # Generate minhashes with pool + tasks = [(get_file_stats, (file,)) for file in files] + + def on_done(_): + return None + + def on_error(_): + return None + + results = pool.map(global_tqdm, tasks, on_error, on_done) + + total_documents, total_size = reduce( + lambda x, y: (x[0] + y[0], x[1] + y[1]), results + ) + + start_offsets = [] + current_offset = 0 + for file_document_count, _ in results: + start_offsets.append(current_offset) + current_offset += file_document_count + + return (total_documents, total_size, start_offsets) + + +if __name__ == "__main__": + version = 1.01 + print(f"Running version {version}") + + stats_file_path = "pile_statistics.json" + if os.path.exists(stats_file_path): + stats = json.load(open(stats_file_path, "r", encoding="utf-8")) + else: + document_count, total_document_size_chars, start_offsets = get_stats() + stats = { + "Data": "Pile statistics", + "Document Count": document_count, + "Total Pile Characters": total_document_size_chars, + "File Start Offsets": start_offsets, + } + json.dump(stats, open(stats_file_path, "w", encoding="utf-8"), indent=4) + + print(f"document_count: {stats['Document Count']}") + print(f"total_chars: {stats['Total Pile Characters']}") + print(f"start_offsets: {stats['File Start Offsets']}") diff --git a/lm-evaluation-harness/scripts/clean_training_data/janitor_util.cpp b/lm-evaluation-harness/scripts/clean_training_data/janitor_util.cpp new file mode 100644 index 0000000000000000000000000000000000000000..858a8b20492507a6228a640cef0cc3ec7ac56bca --- /dev/null +++ b/lm-evaluation-harness/scripts/clean_training_data/janitor_util.cpp @@ -0,0 +1,208 @@ +#include +#include +#include +#include +#include +#include +#include + +bool is_whitespace(char ch) noexcept { + // " \t\n\r\x0b\x0c" (python string.whitespace) + return ch == 32 or (9 <= ch and ch <= 13); + // return ch <= 32; // arguably too general, but slightly faster +} + +bool is_punctuation(char c) noexcept { + // '!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~' ascii values: 33-47, 58-64, + // 91-96, 123-126 + return (33 <= c and c <= 47) or (58 <= c and c <= 64) or + (91 <= c and c <= 96) or (123 <= c and c <= 126); +} + +// Takes a string and makes ngrams of length N, splitting grams on whitespace +// and ignoring ignored characters Returns a LARGE array of ngrams +std::vector clean_ngram(std::string const &input, + std::string const &ignore, + size_t ngram_n) noexcept { + + size_t num_grams = 0; + std::vector ngram_list; + std::vector gram_lengths; + std::string current_ngram; + + // Max gram length is set to 10 below. + current_ngram.reserve(11 * ngram_n); + gram_lengths.reserve(ngram_n); + + bool started_gram = false; + gram_lengths.push_back(0); + + // for (size_t i=0; i 10) { + + // Skip all whitespace + while (++iter != input.end() && is_whitespace(*iter)) + ; + iter--; + + if (started_gram) { + num_grams += 1; + + // Building 1grams is a special case + if (ngram_n == 1) { + ngram_list.push_back(current_ngram); + current_ngram = current_ngram.substr(gram_lengths.front()); + gram_lengths.back() = 0; + + // If there are enough grams to form an ngram, save + } else if (num_grams >= ngram_n) { + // Save the current ngram + ngram_list.push_back(current_ngram); + + // Start the next ngram by dropping the first gram and its space from + // the ngram + current_ngram = current_ngram.substr(gram_lengths.front() + 1); + current_ngram += ' '; + + // Drop the length of the first gram and prepare to record the length + // of the new gram + gram_lengths.erase(gram_lengths.begin()); + gram_lengths.push_back(0); + + // Otherwise, continue building + } else { + current_ngram += ' '; + gram_lengths.push_back(0); + } + + started_gram = false; + } + + // Skip ignored characters + // alternatively, (perhaps marginally) faster: if (is_punctuation(ch)) + // continue; + } else if (ignore.find(*iter) != std::string::npos) { + continue; + } + + // If it is a non-ignored character, add it to the ngram and update the last + // gram's length + else { + current_ngram += tolower(*iter); + gram_lengths.back() += 1; + started_gram = true; + } + } + + return ngram_list; +} + +// Takes a string and makes ngrams of length N, splitting grams on whitespace +// and ignoring ignored characters Returns a LARGE array of tuples of (ngram, +// start_idx, end_idx) +std::vector> +clean_ngram_with_indices(std::string const &input, std::string const &ignore, + size_t ngram_n) noexcept { + + size_t num_grams = 0; + std::vector> ngram_list; + std::vector gram_lengths; + std::vector gram_start_indices; + std::string current_ngram; + + // Max gram length is set to 10 below. + current_ngram.reserve(11 * ngram_n); + + bool started_gram = false; + gram_lengths.push_back(0); + gram_start_indices.push_back(0); + + for (size_t i = 0; i < input.length(); i++) { + char ch = input[i]; + + // If whitespace, end the current ngram and start the next + if (is_whitespace(ch) || gram_lengths.back() > 10) { + + // Skip all whitespace + while (++i < input.length() && is_whitespace(input[i])) + ; + i--; + + if (started_gram) { + num_grams += 1; + + // Building 1grams is a special case + if (ngram_n == 1) { + ngram_list.push_back( + std::make_tuple(current_ngram, gram_start_indices.front(), i)); + current_ngram = current_ngram.substr(gram_lengths.front()); + gram_lengths.back() = 0; + gram_start_indices.back() = i + 1; + + // If there are enough grams to form an ngram, save + } else if (num_grams >= ngram_n) { + + // Save the current ngram + ngram_list.push_back( + std::make_tuple(current_ngram, gram_start_indices.front(), i)); + + // Start the next ngram by dropping the first gram and its space from + // the ngram + current_ngram = current_ngram.substr(gram_lengths.front() + 1); + current_ngram += ' '; + + // Drop the length of the first gram and prepare to record the length + // of the new gram + gram_lengths.erase(gram_lengths.begin()); + gram_lengths.push_back(0); + + gram_start_indices.erase(gram_start_indices.begin()); + gram_start_indices.push_back(i + 1); + + // Otherwise, continue building + } else { + current_ngram += ' '; + gram_lengths.push_back(0); + gram_start_indices.push_back(i + 1); + } + + started_gram = false; + } + + // Skip ignored characters + } else if (ignore.find(ch) != std::string::npos) { + continue; + + // If it is a non-ignored character, add it to the ngram and update the + // last gram's length + } else { + current_ngram += tolower(ch); + gram_lengths.back() += 1; + started_gram = true; + } + } + + return ngram_list; +} + +PYBIND11_MODULE(janitor_util, m) { + m.doc() = "pybind11 example plugin"; // optional module docstring + // m.def("add", &add, "A function which adds two numbers"); // example + // function + m.def("clean_ngram", &clean_ngram, + "Create ngrams of words, ignoring some characters"); + m.def("clean_ngram_with_indices", &clean_ngram_with_indices, + "Create ngrams of words with indices, ignoring some characters"); +} + +// Example compile +// c++ -O3 -Wall -shared -std=c++11 -fPIC $(python3 -m pybind11 --includes) +// janitor_util.cpp -o janitor_util$(python3-config --extension-suffix) If +// python and gcc aren't linked, append to the above: -undefined +// dynamic_lookup diff --git a/lm-evaluation-harness/scripts/clean_training_data/process_sorted_buckets.py b/lm-evaluation-harness/scripts/clean_training_data/process_sorted_buckets.py new file mode 100644 index 0000000000000000000000000000000000000000..9d345d8e86f409495b95a73f4539b2f4df57af70 --- /dev/null +++ b/lm-evaluation-harness/scripts/clean_training_data/process_sorted_buckets.py @@ -0,0 +1,129 @@ +""" +Processes each sorted bucket, creating a new file listing all ngrams that matched more then 10 +unique documents with their unique document counts. Uses multiprocessing and very little memory +as we stream from presorted buckets. Will use a lot of disk though. + +Arguments +--------- +--working_directory (-dir) + Directory containing the sorted buckets, processed files will be deposited here. Default: current directory +--move_dir (-move) + Directory to move processed 13grams too. Default: Do nothing +--process_count (-procs) + Number of processes to use. Default: 4 +""" + +import argparse +import glob +import logging +import os +import re +import shutil +from pathlib import Path + +from tqdm import tqdm +from tqdm_multiprocess import TqdmMultiProcessPool +from tqdm_multiprocess.logger import setup_logger_tqdm + +from scripts.clean_training_data.archiver import TextArchive, TextReader + + +logger = logging.getLogger(__name__) + + +# Multiprocessed +def process_bucket( + bucket_file_path, processed_directory, move_dir, tqdm_func, global_tqdm +): + bucket_id = re.sub("\D", "", os.path.basename(bucket_file_path)) # noqa: W605 + done_file = os.path.join( + processed_directory, f"ngram_bucket_processing_{bucket_id}.done" + ) + if os.path.exists(done_file): + logger.info(f"bucket {bucket_id} already processed, skipping") + return + + # For managing tqdm + file_size = os.path.getsize(bucket_file_path) + bucket_progress = tqdm_func( + total=file_size, dynamic_ncols=True, unit="byte", unit_scale=1 + ) + current_file_position = 0 + update_frequency = 100 * 1000000 # 100mb + update_counter = 0 + + # Iterate through and output ngrams which occur in more then 10 documents + bucket = TextReader(bucket_file_path) + + output_file_path = bucket_file_path + ".processed" + output_archive = TextArchive(output_file_path, mode="wb") + + current_ngram = "" + current_ngram_document_ids = set() + for line in bucket.read(): + [ngram, document_id] = line.rsplit(" ", 1) + + # Write ngram if more then 10 unique document occurrences + if ngram != current_ngram: + if len(current_ngram_document_ids) > 10: + output_archive.add_data( + f"{current_ngram} {len(current_ngram_document_ids)}" + ) + current_ngram = ngram + current_ngram_document_ids = set() + + current_ngram_document_ids.add(document_id) + + # Update tqdm + update_counter += bucket.fh.tell() - current_file_position + current_file_position = bucket.fh.tell() + if update_counter > update_frequency: + bucket_progress.update(update_counter) + update_counter = 0 + + # Remainder + if len(current_ngram_document_ids) > 10: + output_archive.add_data(f"{current_ngram} {len(current_ngram_document_ids)}") + + output_archive.commit() + Path(done_file).touch() + + if move_dir: + shutil.move(output_file_path, move_dir) + + global_tqdm.update() + + +def process_sorted_buckets(working_directory, move_dir, process_count): + bucket_file_paths = glob.glob(os.path.join(working_directory, "*.bkt.txt.sorted")) + processed_directory = os.path.join(working_directory, "processed") + os.makedirs(processed_directory, exist_ok=True) + + pool = TqdmMultiProcessPool(process_count) + tasks = [ + (process_bucket, (bucket_file, processed_directory, move_dir)) + for bucket_file in bucket_file_paths + ] + + global_tqdm = tqdm(total=len(bucket_file_paths), dynamic_ncols=True, unit="bucket") + + def on_done(_): + return None + + def on_error(_): + return None + + _ = pool.map(global_tqdm, tasks, on_error, on_done) + + +parser = argparse.ArgumentParser(description="Process 13 grams from sorted buckets.") +parser.add_argument("-dir", "--working_directory", default="") +parser.add_argument("-move", "--move_dir", default="") +parser.add_argument("-procs", "--process_count", type=int, default=4) + +if __name__ == "__main__": + logfile_path = "process13grams.log" + setup_logger_tqdm(logfile_path) + + args = parser.parse_args() + process_sorted_buckets(args.working_directory, args.move_dir, args.process_count) diff --git a/lm-evaluation-harness/scripts/clean_training_data/sort_13_gram_buckets.py b/lm-evaluation-harness/scripts/clean_training_data/sort_13_gram_buckets.py new file mode 100644 index 0000000000000000000000000000000000000000..83990de822e333bcd16c8d8092aec7ce41ff4e94 --- /dev/null +++ b/lm-evaluation-harness/scripts/clean_training_data/sort_13_gram_buckets.py @@ -0,0 +1,62 @@ +""" +Iteratively runs gnu sort on each bucket, uses up to 8 cores. + +Arguments +--------- +--working_directory (-dir) + Directory containing the bucketed 13-grams. Sorted buckets will be deposited in the same + directory and the unsorted buckets are removed after. +""" + +import argparse +import glob +import logging +import os +import signal +import subprocess +from signal import SIGINT + +from tqdm import tqdm +from tqdm_multiprocess.logger import setup_logger_tqdm + + +logger = logging.getLogger(__name__) + +terminate = False + + +def handler(signal_received, frame): + global terminate + terminate = True + + +def sort_13_gram_buckets(working_directory): + bucket_file_paths = glob.glob(os.path.join(working_directory, "*.bkt.txt")) + + for bucket_file_path in tqdm(bucket_file_paths, dynamic_ncols=True): + sorted_file_path = bucket_file_path + ".sorted" + command = f"sort {bucket_file_path} > {sorted_file_path}" + logger.info(command) + subprocess.call(command, shell=True) + + if terminate: + return + + os.remove(bucket_file_path) + + +parser = argparse.ArgumentParser(description="sort 13gram buckets") +parser.add_argument("-dir", "--working_directory", default="") + +if __name__ == "__main__": + version = 1.00 + print(f"Running version {version}") + + # Handle sigint (ctrl-c) cleanly + previous_signal_int = signal.signal(SIGINT, handler) + + logfile_path = "sort13grambuckets.log" + setup_logger_tqdm(logfile_path) + + args = parser.parse_args() + sort_13_gram_buckets(args.working_directory) diff --git a/lm-evaluation-harness/scripts/get_prompts.py b/lm-evaluation-harness/scripts/get_prompts.py new file mode 100644 index 0000000000000000000000000000000000000000..d262ec37e40f229c2009f9f162cc58834291de12 --- /dev/null +++ b/lm-evaluation-harness/scripts/get_prompts.py @@ -0,0 +1,25 @@ +from itertools import islice + +from lm_eval import tasks + + +ct = 3 + +for ( + tname, + Task, +) in tasks.TASK_REGISTRY.items(): # [('record', tasks.superglue.ReCoRD)]:# + task = Task() + + print("#", tname) + docs = islice( + task.validation_docs() if task.has_validation_docs() else task.test_docs(), ct + ) + print() + for i in range(ct): + print() + doc = next(docs) + print("**Context**:", "\n```\n" + task.doc_to_text(doc) + "\n```\n") + print() + print("**Target**:", "\n```\n" + task.doc_to_target(doc) + "\n```\n") + print() diff --git a/lm-evaluation-harness/scripts/make_gpt2_test_cases.py b/lm-evaluation-harness/scripts/make_gpt2_test_cases.py new file mode 100644 index 0000000000000000000000000000000000000000..0c1a4bffe03ef057c331dc9a20c0a5eadb46be66 --- /dev/null +++ b/lm-evaluation-harness/scripts/make_gpt2_test_cases.py @@ -0,0 +1,48 @@ +import random + +import torch +import torch.nn.functional as F +import transformers + + +random.seed(42) + + +data = [ + "A multilayer perceptron (MLP) is a class of feedforward artificial neural network (ANN)", + "The term MLP is used ambiguously, sometimes loosely to any feedforward ANN, sometimes strictly to refer to networks composed of multiple layers of perceptrons (with threshold activation); see § Terminology", + 'Multilayer perceptrons are sometimes colloquially referred to as "vanilla" neural networks, especially when they have a single hidden layer.[1]', + "An MLP consists of at least three layers of nodes: an input layer, a hidden layer and an output layer. Except for the input nodes, each node is a neuron that uses a nonlinear activation function.", + "MLP utilizes a supervised learning technique called backpropagation for training.[2][3] Its multiple layers and non-linear activation distinguish MLP from a linear perceptron. It can distinguish data that is not linearly separable.[4]", + "Recent work has demonstrated substantial gains on many NLP tasks and benchmarks by pre-training on a large corpus of text followed by fine-tuning on a specific task. While typically task-agnostic in architecture, this method still requires task-specific fine-tuning datasets of thousands or tens of thousands of examples. By contrast, humans can generally perform a new language task from only a few examples or from simple instructions - something which current NLP systems still largely struggle to do. Here we show that scaling up language models greatly improves task-agnostic, few-shot performance, sometimes even reaching competitiveness with prior state-of-the-art fine-tuning approaches. ", + "Specifically, we train GPT-3, an autoregressive language model with 175 billion parameters, 10x more than any previous non-sparse language model, and test its performance in the few-shot setting. For all tasks, GPT-3 is applied without any gradient updates or fine-tuning, with tasks and few-shot demonstrations specified purely via text interaction with the model. GPT-3 achieves strong performance on many NLP datasets, including translation, question-answering, and cloze tasks, as well as several tasks that require on-the-fly reasoning or domain adaptation, such as unscrambling words, using a novel word in a sentence, or performing 3-digit arithmetic. At the same time, we also identify some datasets where GPT-3's few-shot learning still struggles, as well as some datasets where GPT-3 faces methodological issues related to training on large web corpora. Finally, we find that GPT-3 can generate samples of news articles which human evaluators have difficulty distinguishing from articles written by humans. We discuss broader societal impacts of this finding and of GPT-3 in general.", + "A multilayer perceptron (MLP) is a class of feedforward artificial neural network (ANN)", + "Hello World", +] + + +model = transformers.GPT2LMHeadModel.from_pretrained("gpt2") +tok = transformers.GPT2Tokenizer.from_pretrained("gpt2") + +tgs = [] + +for dat in data: + random.seed(dat) + # print(model(tok.encode(dat, return_tensors="pt"))[0][0]) + + toks = tok.encode(dat, return_tensors="pt") + ind = random.randrange(len(toks[0]) - 1) + logits = F.log_softmax(model(toks)[0], dim=-1)[:, :-1] # [batch, seq, vocab] + + res = torch.gather(logits, 2, toks[:, 1:].unsqueeze(-1)).squeeze(-1)[0] + + tgs.append(float(res[ind:].sum())) + print( + r'("""' + + tok.decode(toks[0, : ind + 1]) + + r'""", """' + + tok.decode(toks[0, ind + 1 :]) + + r'"""), ' + ) + +print(tgs) diff --git a/lm-evaluation-harness/scripts/make_table_results.py b/lm-evaluation-harness/scripts/make_table_results.py new file mode 100644 index 0000000000000000000000000000000000000000..1bf1dcab5c1eacc61a4cd789186590ff65f5fa4f --- /dev/null +++ b/lm-evaluation-harness/scripts/make_table_results.py @@ -0,0 +1,74 @@ +""" +Usage: + python make_table_tasks.py --output +""" + +import json +import logging +import os + +from pytablewriter import LatexTableWriter, MarkdownTableWriter + + +logger = logging.getLogger(__name__) + + +def make_table(result_dict): + """Generate table of results.""" + md_writer = MarkdownTableWriter() + latex_writer = LatexTableWriter() + md_writer.headers = ["Task", "Version", "Metric", "Value", "", "Stderr"] + latex_writer.headers = ["Task", "Version", "Metric", "Value", "", "Stderr"] + + values = [] + + for k, dic in sorted(result_dict["results"].items()): + version = result_dict["versions"][k] + percent = k == "squad2" + for m, v in dic.items(): + if m.endswith("_stderr"): + continue + + if m + "_stderr" in dic: + se = dic[m + "_stderr"] + if percent or m == "ppl": + values.append([k, version, m, "%.2f" % v, "±", "%.2f" % se]) + else: + values.append( + [k, version, m, "%.2f" % (v * 100), "±", "%.2f" % (se * 100)] + ) + else: + if percent or m == "ppl": + values.append([k, version, m, "%.2f" % v, "", ""]) + else: + values.append([k, version, m, "%.2f" % (v * 100), "", ""]) + k = "" + version = "" + md_writer.value_matrix = values + latex_writer.value_matrix = values + + # todo: make latex table look good + # print(latex_writer.dumps()) + + return md_writer.dumps() + + +if __name__ == "__main__": + # loop dirs and subdirs in results dir + # for each dir, load json files + for dirpath, dirnames, filenames in os.walk("../results"): + # skip dirs without files + if not filenames: + continue + path_readme = os.path.join(dirpath, "README.md") + with open(path_readme, "w", encoding="utf-8") as f: + # get path name, only last folder + path_name = dirpath.split("/")[-1] + f.write(f"# {path_name} \n\n") + for filename in sorted([f for f in filenames if f.endswith(".json")]): + path = os.path.join(dirpath, filename) + with open(path, "r", encoding="utf-8") as f: + result_dict = json.load(f) + with open(path_readme, "a", encoding="utf-8") as f: + f.write(f"## {filename} \n") + f.write(f"{make_table(result_dict)} \n") diff --git a/lm-evaluation-harness/scripts/make_table_tasks.py b/lm-evaluation-harness/scripts/make_table_tasks.py new file mode 100644 index 0000000000000000000000000000000000000000..4e53cef10d9da66600aac3c95e57c2ee406336f2 --- /dev/null +++ b/lm-evaluation-harness/scripts/make_table_tasks.py @@ -0,0 +1,54 @@ +""" +Usage: + python make_table_tasks.py --output +""" + +import argparse +import logging + +from pytablewriter import MarkdownTableWriter + +from lm_eval import tasks + + +logger = logging.getLogger(__name__) + + +def check(tf): + if tf: + return "✓" + else: + return " " + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--output", type=str, default="task_table.md") + args = parser.parse_args() + + writer = MarkdownTableWriter() + writer.headers = ["Task Name", "Train", "Val", "Test", "Val/Test Docs", "Metrics"] + values = [] + + tasks = tasks.TASK_REGISTRY.items() + tasks = sorted(tasks, key=lambda x: x[0]) + for tname, Task in tasks: + task = Task() + v = [ + tname, + check(task.has_training_docs()), + check(task.has_validation_docs()), + check(task.has_test_docs()), + len( + list( + task.test_docs() if task.has_test_docs() else task.validation_docs() + ) + ), + ", ".join(task.aggregation().keys()), + ] + logger.info(v) + values.append(v) + writer.value_matrix = values + table = writer.dumps() + with open(args.output, "w", encoding="utf-8") as f: + f.write(table) diff --git a/lm-evaluation-harness/scripts/model_comparator.py b/lm-evaluation-harness/scripts/model_comparator.py new file mode 100644 index 0000000000000000000000000000000000000000..258788d4d0c1c2b8901317ea63ef66a55ca67987 --- /dev/null +++ b/lm-evaluation-harness/scripts/model_comparator.py @@ -0,0 +1,140 @@ +import argparse +import logging +import os +from typing import Dict, List, Tuple + +import numpy as np +import pandas as pd +import torch + +import lm_eval.evaluator +import lm_eval.models.utils +from lm_eval import tasks + + +os.environ["TOKENIZERS_PARALLELISM"] = "false" +eval_logger = logging.getLogger(__name__) + + +def memory_stats(): + eval_logger.info( + f"Memory allocated: {torch.cuda.memory_allocated() / 1024**2}, reserved: {torch.cuda.memory_reserved() // 1024**2}" + ) + + +def calculate_z_value(res1: Dict, res2: Dict) -> Tuple[float, float]: + from scipy.stats.norm import sf + + acc1, acc2 = res1["acc,none"], res2["acc,none"] + st_err1, st_err2 = res1["acc_stderr,none"], res2["acc_stderr,none"] + Z = (acc1 - acc2) / np.sqrt((st_err1**2) + (st_err2**2)) + # Determining the p-value + p_value = 2 * sf(abs(Z)) # two-tailed test + return Z, p_value + + +def print_results( + data_to_print: List = None, results_dict: Dict = None, alpha: float = None +): + model1_data = data_to_print[0] + model2_data = data_to_print[1] + table_data = [] + for task in model1_data.keys(): + row = { + "Task": task, + "HF Accuracy": model1_data[task]["acc,none"], + "vLLM Accuracy": model2_data[task]["acc,none"], + "HF StdErr": model1_data[task]["acc_stderr,none"], + "vLLM StdErr": model2_data[task]["acc_stderr,none"], + } + table_data.append(row) + comparison_df = pd.DataFrame(table_data) + comparison_df["Z-Score"] = comparison_df["Task"].apply( + lambda task: results_dict[task]["z"] + ) + comparison_df["P-Value"] = comparison_df["Task"].apply( + lambda task: results_dict[task]["p_value"] + ) + comparison_df[f"p > {alpha}"] = comparison_df["P-Value"].apply( + lambda p: "✓" if p > alpha else "×" + ) + return comparison_df + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--pretrained", default="EleutherAI/pythia-70m", help="name of model to compare" + ) + parser.add_argument( + "--hf_args", help="huggingface model args =", default="" + ) + parser.add_argument("--vllm_args", help="vllm model args =", default="") + parser.add_argument("--tasks", type=str, default="arc_easy,hellaswag") + parser.add_argument( + "--limit", + type=float, + default=100, + ) + parser.add_argument( + "--alpha", + type=float, + default=0.05, + help="Significance level for two-tailed z-test", + ) + parser.add_argument( + "--device", + type=str, + default="cuda", + ) + parser.add_argument( + "--batch", + type=str, + default=8, + ) + parser.add_argument( + "--verbosity", + type=str, + default="INFO", + help="Logging verbosity", + ) + return parser.parse_args() + + +if __name__ == "__main__": + tasks.initialize_tasks() + args = parse_args() + tasks = args.tasks.split(",") + print(tasks) + hf_args, vllm_args = "," + args.hf_args, "," + args.vllm_args + results_vllm = lm_eval.evaluator.simple_evaluate( + model="vllm", + model_args=f"pretrained={args.pretrained}" + vllm_args, + tasks=tasks, + limit=args.limit, + device=args.device, + batch_size=args.batch, + ) + memory_stats() + lm_eval.models.utils.clear_torch_cache() + eval_logger.info("Memory stats cleared") + memory_stats() + results_hf = lm_eval.evaluator.simple_evaluate( + model="hf", + model_args=f"pretrained={args.pretrained}" + hf_args, + tasks=tasks, + limit=args.limit, + device=args.device, + batch_size=args.batch, + ) + all_res = {} + for task1, task2 in zip( + results_hf["results"].items(), results_vllm["results"].items() + ): + assert task1[0] == task2[0] + z, p_value = calculate_z_value(task1[1], task2[1]) + all_res[task1[0]] = {"z": z, "p_value": p_value} + df = print_results( + [results_hf["results"], results_vllm["results"]], all_res, args.alpha + ) + print(df) diff --git a/lm-evaluation-harness/scripts/regression.py b/lm-evaluation-harness/scripts/regression.py new file mode 100644 index 0000000000000000000000000000000000000000..75258dcb640a4f32a0011e864d390e9619f6e2e3 --- /dev/null +++ b/lm-evaluation-harness/scripts/regression.py @@ -0,0 +1,199 @@ +import argparse +import json +import os +import subprocess +import time +from pathlib import Path + +from lm_eval import utils +from lm_eval.api.registry import ALL_TASKS + + +seq2seq_models = ["google/flan-t5-small"] +causal_models = [ + "gpt2", + "facebook/opt-125m", + "EleutherAI/gpt-neo-125m", + "EleutherAI/pythia-160m", +] +model_names = seq2seq_models + causal_models + + +completion_tasks = ["boolq", "lambada_openai", "winogrande"] +choice_tasks = ["hellaswag", "openbookqa", "piqa"] +perplexity_tasks = ["wikitext"] +generation_tasks = [] +task_names = completion_tasks + choice_tasks + perplexity_tasks + generation_tasks + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--branches", default=[]) + parser.add_argument("--models", default=model_names) + parser.add_argument("--tasks", default=task_names) + parser.add_argument("--acc_norm", type=bool, default=False) + parser.add_argument("--perplexity", default=None) + # TODO: implement num_fewshot and limit per task, e.g. task1:5,task2:1:100,task3::1000 + parser.add_argument("--num_fewshot", type=int, default=0) + parser.add_argument("--limit", type=float, default=None) + # TODO: implement hf-auto to pick between causal and seq2seq models so we don't need this + parser.add_argument("--model", default="hf-causal") + # Use whatever is faster here + parser.add_argument("--model_args", default="use_accelerate=True,load_in_8bit=True") + parser.add_argument("--batch_size", default="auto") + return parser.parse_args() + + +def eval_models(args, branch=None): + if branch is not None: + if os.system(f"git checkout {branch}") != 0: + return {}, 0 + + branch = branch or initial_branch + + start_time = time.time() + + results = {} + + for model in args.models: + model_type = ( + "hf-causal" + if model in causal_models + else "hf-seq2seq" + if model in seq2seq_models + else args.model + ) + model_args = f"pretrained={model},{args.model_args}" + # TODO: split_and_pad_windows in AutoSeq2SeqLM doesn"t exist, #527 + tasks = ( + args.tasks + if model in causal_models or model_type == "hf-causal" + else list(filter(lambda task: task not in perplexity_tasks, args.tasks)) + ) + # TODO: OOM with auto for seq2seq models, also can OOM with llama + batch_size = ( + args.batch_size + if model in causal_models or model_type == "hf-causal" + else 64 + if args.batch_size == "auto" + else args.batch_size + ) + output_path = ( + f"data/regression/{int(start_time)}-{branch}-{Path(model).name}.json" + ) + + command = ( + f"python3 main.py --model {model_type} --model_args {model_args} --tasks {','.join(tasks)} " + f"--num_fewshot {args.num_fewshot}{'' if args.limit is None else f' --limit {args.limit}'} " + f"--batch_size {batch_size} --no_cache --output_path {output_path}" + ) + + print( + f"{'=' * 80}\nEvaluating {model} on {', '.join(tasks)} at {branch} with:\n\n{command}\n{'=' * 80}" + ) + + ret = os.system(command) + + results[model] = ( + json.load(open(output_path, encoding="utf-8")) + if ret == 0 + else {"results": {}} + ) + + end_time = time.time() + + return results, end_time - start_time + + +def extract_value(args, results, model, task, err=False): + if model not in results: + return 0 + results = results[model]["results"] + if task not in results: + return 0 + results = results[task] + if args.acc_norm and "acc_norm,none" in results: + return results["acc_norm,none"] if not err else results["acc_norm_stderr,none"] + if "acc,none" in results: + return results["acc,none"] if not err else results["acc_stderr,none"] + if (args.perplexity or "word_perplexity") + ",none" in results: + return ( + results[(args.perplexity or "word_perplexity") + ",none"] if not err else 0 + ) + return 0 + + +def format_value(args, results, model, task): + val = 100 * extract_value(args, results, model, task) + err = 100 * extract_value(args, results, model, task, err=True) + return f"{val:.2f}{f' ± {err:.2f}' if err != 0 else ''}" + + +def format_diff(args, results1, results2, model, task): + val1 = 100 * extract_value(args, results1, model, task) + val2 = 100 * extract_value(args, results2, model, task) + diff = val2 - val1 + return f"**+{diff:.2f}**" if diff > 0 else f"{diff:.2f}" + + +def main(): + args = parse_args() + + args.branches = ( + args.branches.split(",") if isinstance(args.branches, str) else args.branches + ) + args.models = ( + args.models.split(",") if isinstance(args.models, str) else args.models + ) + args.tasks = ( + ALL_TASKS + if args.tasks == "all_tasks" + else utils.pattern_match(args.tasks.split(","), ALL_TASKS) + if isinstance(args.tasks, str) + else args.tasks + ) + + global initial_branch + initial_branch = ( + subprocess.check_output("git branch --show-current", shell=True) + .decode("ascii") + .strip() + ) + + # TODO: implement proper timing for each task + # TODO: reduce IO by sharing tasks between models? + + results, runtime = eval_models(args) + print(results, runtime) + + runs = [] + for branch in args.branches: + runs.append((branch, *eval_models(args, branch))) + + os.system(f"git checkout {initial_branch}") + + print("") + print(f"|task|{'|'.join(map(lambda model: Path(model).name, args.models))}|") + print(f"|--|{'--|' * len(args.models)}") + for task in args.tasks: + print( + f"|{task} ({initial_branch})|{'|'.join(map(lambda model: format_value(args, results, model, task), args.models))}|" + ) + for branch, branch_results, branch_runtime in runs: + print( + f"|{task} ({branch})|{'|'.join(map(lambda model: format_value(args, branch_results, model, task), args.models))}|" + ) + print( + f"|{task} (diff)|{'|'.join(map(lambda model: format_diff(args, results, branch_results, model, task), args.models))}|" + ) + + print("") + print("|branch|runtime|%|") + print("|--|--|--|") + print(f"|{initial_branch}|{runtime:.1f}s|100%|") + for branch, _, branch_runtime in runs: + print(f"|{branch}|{branch_runtime:.1f}s|{100 * branch_runtime / runtime:.2f}%|") + + +if __name__ == "__main__": + main() diff --git a/lm-evaluation-harness/scripts/requests_caching.py b/lm-evaluation-harness/scripts/requests_caching.py new file mode 100644 index 0000000000000000000000000000000000000000..c75721e36b87650b8c021f5b16fca4fe4b529c50 --- /dev/null +++ b/lm-evaluation-harness/scripts/requests_caching.py @@ -0,0 +1,95 @@ +""" +Usage: + python requests_caching.py --tasks=comma,separated,list,of,tasks --cache_requests= +""" + +import argparse +import logging +import os +from typing import List + +import torch +from transformers import ( + pipeline as trans_pipeline, +) + +from lm_eval import simple_evaluate +from lm_eval.evaluator import request_caching_arg_to_dict + + +eval_logger = logging.getLogger(__name__) + + +MODULE_DIR = os.path.dirname(os.path.realpath(__file__)) + +# Used to specify alternate cache path, useful if run in a docker container +# NOTE raw datasets will break if you try to transfer the cache from your host to a docker image +LM_HARNESS_CACHE_PATH = os.getenv("LM_HARNESS_CACHE_PATH") + + +DEVICE = "cuda" if torch.cuda.is_available() else "cpu" + +MODEL = "EleutherAI/pythia-70m" + +TASK = "text-generation" + + +def run_model_for_task_caching(tasks: List[str], cache_requests: str): + eval_logger.info(f"Loading HF model: {MODEL}") + + trans_pipe = trans_pipeline( + task=TASK, model=MODEL, device=DEVICE, trust_remote_code=True + ) + + model = trans_pipe.model + tokenizer = trans_pipe.tokenizer + + eval_logger.info( + f"Running simple_evaluate to cache request objects for tasks: {tasks}" + ) + + cache_args = request_caching_arg_to_dict(cache_requests=cache_requests) + + eval_logger.info( + f"The following operations will be performed on the cache: {cache_requests}" + ) + + eval_data = simple_evaluate( + model="hf-auto", + model_args={ + "pretrained": model, + "tokenizer": tokenizer, + }, + limit=1, + device=DEVICE, + tasks=tasks, + write_out=True, + **cache_args, + ) + + return eval_data + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--tasks", + "-t", + default=None, + metavar="task1,task2", + ) + parser.add_argument( + "--cache_requests", + type=str, + default=None, + choices=["true", "refresh", "delete"], + help="Speed up evaluation by caching the building of dataset requests. `None` if not caching.", + ) + + args = parser.parse_args() + + tasks = args.tasks.split(",") + + eval_data = run_model_for_task_caching( + tasks=tasks, model=MODEL, device=DEVICE, cache_requests=args.cache_requests + ) diff --git a/lm-evaluation-harness/scripts/write_out.py b/lm-evaluation-harness/scripts/write_out.py new file mode 100644 index 0000000000000000000000000000000000000000..60faf22ff31a1b5d958d476858afc22347d55305 --- /dev/null +++ b/lm-evaluation-harness/scripts/write_out.py @@ -0,0 +1,101 @@ +import argparse +import logging +import os +import random + +import numpy as np + +from lm_eval import tasks +from lm_eval.evaluator_utils import get_task_list +from lm_eval.tasks import TaskManager +from lm_eval.utils import join_iters + + +eval_logger = logging.getLogger(__name__) + + +EXAMPLE_DIVIDER = "!!@@##@@!! -- Example {i}\n" + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--output_base_path", "--output_path", required=True) + parser.add_argument("--tasks", default="all_tasks") + parser.add_argument("--sets", type=str, default="val") # example: val,test + parser.add_argument("--num_fewshot", type=int, default=1) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--num_examples", type=int, default=1) + parser.add_argument( + "--include_path", + type=str, + default=None, + help="Additional path to include if there are external tasks to include.", + ) + parser.add_argument( + "--verbosity", + type=str, + default="INFO", + help="Log error when tasks are not registered.", + ) + return parser.parse_args() + + +def main(): + args = parse_args() + np.random.seed(args.seed) + + if args.include_path is not None: + eval_logger.info(f"Including path: {args.include_path}") + + task_manager = TaskManager(args.verbosity, include_path=args.include_path) + + if args.tasks == "all_tasks": + task_names = task_manager.all_tasks + else: + task_names = args.tasks.split(",") + task_dict = tasks.get_task_dict(task_names, task_manager) + + os.makedirs(args.output_base_path, exist_ok=True) + for task in [x.task for x in get_task_list(task_dict)]: + task_name = task.config.task + rnd = random.Random() + rnd.seed(args.seed) + + iters = [] + + for set in args.sets.split(","): + docs = None + if set == "train" and task.has_training_docs(): + docs = task.training_docs() + if set == "val" and task.has_validation_docs(): + docs = task.validation_docs() + if set == "test" and task.has_test_docs(): + docs = task.test_docs() + if docs is not None: + iters.append(docs) + + if len(iters) == 0: + raise ValueError( + f"Passed --sets '{args.sets}' but this task has no splits which match. Please specify a different --sets value." + ) + + docs = join_iters(iters) + + with open( + os.path.join(args.output_base_path, task_name), "w", encoding="utf8" + ) as f: + for i, doc in ( + zip(range(args.num_examples), docs) + if args.num_examples > 0 + else enumerate(docs) + ): + f.write(EXAMPLE_DIVIDER.format(i=i)) + ctx = task.fewshot_context( + doc=doc, + num_fewshot=args.num_fewshot, + ) + f.write(ctx + "\n") + + +if __name__ == "__main__": + main() diff --git a/lm-evaluation-harness/scripts/zeno_visualize.py b/lm-evaluation-harness/scripts/zeno_visualize.py new file mode 100644 index 0000000000000000000000000000000000000000..2c8e1423726579880220d5055973fb4d22faa4db --- /dev/null +++ b/lm-evaluation-harness/scripts/zeno_visualize.py @@ -0,0 +1,257 @@ +import argparse +import json +import logging +import os +import re +from pathlib import Path + +import pandas as pd +from zeno_client import ZenoClient, ZenoMetric + +from lm_eval.utils import ( + get_latest_filename, + get_results_filenames, + get_sample_results_filenames, +) + + +eval_logger = logging.getLogger(__name__) + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Upload your data to the Zeno AI evaluation platform to visualize results. This requires a ZENO_API_KEY in your environment variables. The eleuther harness must be run with log_samples=True and an output_path set for data to be written to disk." + ) + parser.add_argument( + "--data_path", + required=True, + help="Where to find the results of the benchmarks that have been run. Uses the name of each subfolder as the model name.", + ) + parser.add_argument( + "--project_name", + required=True, + help="The name of the generated Zeno project.", + ) + return parser.parse_args() + + +def main(): + """Upload the results of your benchmark tasks to the Zeno AI evaluation platform. + + This scripts expects your results to live in a data folder where subfolders contain results of individual models. + """ + args = parse_args() + + client = ZenoClient(os.environ["ZENO_API_KEY"]) + + # Get all model subfolders from the parent data folder. + models = [ + os.path.basename(os.path.normpath(f)) + for f in os.scandir(Path(args.data_path)) + if f.is_dir() + ] + + assert len(models) > 0, "No model directories found in the data_path." + + # Get the tasks from the latest results file of the first model. + tasks = set(tasks_for_model(models[0], args.data_path)) + + # Get tasks names from the latest results file for each model + # Get intersection of tasks for all models + for model in models: + old_tasks = tasks.copy() + task_count = len(tasks) + model_tasks = set(tasks_for_model(model, args.data_path)) + tasks.intersection(set(model_tasks)) + + if task_count != len(tasks): + eval_logger.warning( + f"All models must have the same tasks. {model} has tasks: {model_tasks} but have already recorded tasks: {old_tasks}. Taking intersection {tasks}" + ) + + assert len(tasks) > 0, ( + "Must provide at least one task in common amongst models to compare." + ) + + for task in tasks: + # Upload data for all models + for model_index, model in enumerate(models): + # Get latest results and sample results for a model + model_dir = Path(args.data_path, model) + model_files = [f.as_posix() for f in model_dir.iterdir() if f.is_file()] + model_results_filenames = get_results_filenames(model_files) + model_sample_filenames = get_sample_results_filenames(model_files) + latest_results = get_latest_filename( + [Path(f).name for f in model_results_filenames] + ) + latest_sample_results = get_latest_filename( + [Path(f).name for f in model_sample_filenames if task in f] + ) + model_args = re.sub( + r"[\"<>:/\|\\?\*\[\]]+", + "__", + json.load( + open(Path(args.data_path, model, latest_results), encoding="utf-8") + )["config"]["model_args"], + ) + print(model_args) + data = [] + with open( + Path(args.data_path, model, latest_sample_results), + "r", + encoding="utf-8", + ) as file: + for line in file: + data.append(json.loads(line.strip())) + + configs = json.load( + open(Path(args.data_path, model, latest_results), encoding="utf-8") + )["configs"] + config = configs[task] + + if model_index == 0: # Only need to assemble data for the first model + metrics = [] + for metric in config["metric_list"]: + if metric.get("aggregation") == "mean": + metrics.append( + ZenoMetric( + name=metric["metric"], + type="mean", + columns=[metric["metric"]], + ) + ) + project = client.create_project( + name=args.project_name + (f"_{task}" if len(tasks) > 1 else ""), + view="text-classification", + metrics=metrics, + ) + project.upload_dataset( + generate_dataset(data, config), + id_column="id", + data_column="data", + label_column="labels", + ) + + project.upload_system( + generate_system_df(data, config), + name=model, + id_column="id", + output_column="output", + ) + + +def tasks_for_model(model: str, data_path: str): + """Get the tasks for a specific model. + + Args: + model (str): The name of the model. + data_path (str): The path to the data. + + Returns: + list: A list of tasks for the model. + """ + # get latest model results for a given name + model_dir = Path(data_path, model) + model_files = [f.as_posix() for f in model_dir.iterdir() if f.is_file()] + model_results_filenames = get_results_filenames(model_files) + latest_results = get_latest_filename(model_results_filenames) + config = (json.load(open(latest_results, encoding="utf-8"))["configs"],) + return list(config[0].keys()) + + +def generate_dataset( + data, + config, +): + """Generate a Zeno dataset from evaluation data. + + Args: + data: The data to generate a dataset for. + config: The configuration of the task. + + Returns: + pd.Dataframe: A dataframe that is ready to be uploaded to Zeno. + """ + ids = ( + [x["doc_id"] for x in data] + if not config.get("filter_list") + else [f"{x['doc_id']}.{x['filter']}" for x in data] + ) + labels = [x["target"] for x in data] + instance = [""] * len(ids) + + if config["output_type"] == "loglikelihood": + instance = [x["arguments"]["gen_args_0"]["arg_0"] for x in data] + labels = [x["arguments"]["gen_args_0"]["arg_1"] for x in data] + elif config["output_type"] == "multiple_choice": + instance = [ + x["arguments"]["gen_args_0"]["arg_0"] + + "\n\n" + + "\n".join([f"- {y[1]}" for y in x["arguments"]]) + for x in data + ] + elif config["output_type"] == "loglikelihood_rolling": + instance = [x["arguments"]["gen_args_0"]["arg_0"] for x in data] + elif config["output_type"] == "generate_until": + instance = [x["arguments"]["gen_args_0"]["arg_0"] for x in data] + + return pd.DataFrame( + { + "id": ids, + "doc_id": [x["doc_id"] for x in data], + "data": instance, + "input_len": [len(x) for x in instance], + "labels": labels, + "output_type": config["output_type"], + } + ) + + +def generate_system_df(data, config): + """Generate a dataframe for a specific system to be uploaded to Zeno. + + Args: + data: The data to generate a dataframe from. + config: The configuration of the task. + + Returns: + pd.Dataframe: A dataframe that is ready to be uploaded to Zeno as a system. + """ + ids = ( + [x["doc_id"] for x in data] + if not config.get("filter_list") + else [f"{x['doc_id']}.{x['filter']}" for x in data] + ) + system_dict = {"id": ids} + system_dict["doc_id"] = [x["doc_id"] for x in data] + if config.get("filter_list"): + system_dict["filter"] = [x["filter"] for x in data] + system_dict["output"] = [""] * len(ids) + + if config["output_type"] == "loglikelihood": + system_dict["output"] = [ + "correct" if x["filtered_resps"][0][1] is True else "incorrect" + for x in data + ] + elif config["output_type"] == "multiple_choice": + system_dict["output"] = [ + ", ".join([str(y[0]) for y in x["filtered_resps"]]) for x in data + ] + system_dict["num_answers"] = [len(x["filtered_resps"]) for x in data] + elif config["output_type"] == "loglikelihood_rolling": + system_dict["output"] = [str(x["filtered_resps"][0]) for x in data] + elif config["output_type"] == "generate_until": + system_dict["output"] = [str(x["filtered_resps"][0]) for x in data] + system_dict["output_length"] = [len(str(x["filtered_resps"][0])) for x in data] + + metrics = { + metric["metric"]: [x[metric["metric"]] for x in data] + for metric in config["metric_list"] + } + system_dict.update(metrics) + system_df = pd.DataFrame(system_dict) + return system_df + + +if __name__ == "__main__": + main() diff --git a/lm-evaluation-harness/scripts_cxr/eval.sh b/lm-evaluation-harness/scripts_cxr/eval.sh new file mode 100644 index 0000000000000000000000000000000000000000..e3baf8d0434c5e5c4b160428b2fe158eef62123e --- /dev/null +++ b/lm-evaluation-harness/scripts_cxr/eval.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# tasks=triviaqa +tasks=(hellaswag rte mmlu) +models=('Llama-2-7b-hf' 'Llama-2-7b-qint4' 'Llama-2-7b-qint8' 'Llama-2-7b-qint-top_m_alpha_values') + +# tasks=winogrande +for task in "${tasks[@]}"; do + mkdir -p results/${task} + for model in "${models[@]}"; do + echo $model + lm_eval --model hf \ + --model_args pretrained=../models/${model} \ + --tasks $task \ + --device cuda:0 \ + --batch_size 8 \ + --output_path results/${task}/${model}.json + + done +done \ No newline at end of file diff --git a/lm-evaluation-harness/templates/new_yaml_task/README.md b/lm-evaluation-harness/templates/new_yaml_task/README.md new file mode 100644 index 0000000000000000000000000000000000000000..4ca7b63a59139e742407ee43250841f662530c91 --- /dev/null +++ b/lm-evaluation-harness/templates/new_yaml_task/README.md @@ -0,0 +1,48 @@ +# Task-name + +## Paper + +Title: `paper titles goes here` + +Abstract: `link to paper PDF or arXiv abstract goes here` + +`Short description of paper / benchmark goes here:` + +Homepage: `homepage to the benchmark's website goes here, if applicable` + +### Citation + +```text +BibTeX-formatted citation goes here +``` + +### Groups, Tags, and Tasks + +#### Groups + +* `group_name`: `Short description` + +#### Tags + +* `tag_name`: `Short description` + +#### Tasks + +* `task_name`: `1-sentence description of what this particular task does` +* `task_name2`: ... + +### Checklist + +For adding novel benchmarks/datasets to the library: + +* [ ] Is the task an existing benchmark in the literature? + * [ ] Have you referenced the original paper that introduced the task? + * [ ] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + +If other tasks on this dataset are already supported: + +* [ ] Is the "Main" variant of this task clearly denoted? +* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [ ] Have you noted which, if any, published evaluation setups are matched by this variant? + +### Changelog diff --git a/lm-evaluation-harness/templates/new_yaml_task/blank_yaml.yaml b/lm-evaluation-harness/templates/new_yaml_task/blank_yaml.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lm-evaluation-harness/tests/__init__.py b/lm-evaluation-harness/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lm-evaluation-harness/tests/models/test_api.py b/lm-evaluation-harness/tests/models/test_api.py new file mode 100644 index 0000000000000000000000000000000000000000..2db223774702eca0dfa245a9de4e43b40bf6ba75 --- /dev/null +++ b/lm-evaluation-harness/tests/models/test_api.py @@ -0,0 +1,228 @@ +import asyncio +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from lm_eval.models.openai_completions import LocalCompletionsAPI + + +@pytest.fixture +def api(): + return LocalCompletionsAPI( + base_url="http://test-url.com", tokenizer_backend=None, model="gpt-3.5-turbo" + ) + + +@pytest.fixture +def api_tokenized(): + return LocalCompletionsAPI( + base_url="http://test-url.com", + model="EleutherAI/pythia-1b", + tokenizer_backend="huggingface", + ) + + +@pytest.fixture +def api_batch_ssl_tokenized(): + return LocalCompletionsAPI( + base_url="https://test-url.com", + model="EleutherAI/pythia-1b", + verify_certificate=False, + num_concurrent=2, + tokenizer_backend="huggingface", + ) + + +def test_create_payload_generate(api): + messages = ["Generate a story"] + gen_kwargs = { + "max_tokens": 100, + "temperature": 0.7, + "until": ["The End"], + "do_sample": True, + "seed": 1234, + } + payload = api._create_payload(messages, generate=True, gen_kwargs=gen_kwargs) + + assert payload == { + "prompt": ["Generate a story"], + "model": "gpt-3.5-turbo", + "max_tokens": 100, + "temperature": 0.7, + "stop": ["The End"], + "seed": 1234, + } + + +def test_create_payload_loglikelihood(api): + messages = ["The capital of France is"] + payload = api._create_payload(messages, generate=False, gen_kwargs=None) + + assert payload == { + "model": "gpt-3.5-turbo", + "prompt": ["The capital of France is"], + "max_tokens": 1, + "logprobs": 1, + "echo": True, + "temperature": 0, + "seed": 1234, + } + + +@pytest.mark.parametrize( + "input_messages, generate, gen_kwargs, expected_payload", + [ + ( + ["Hello, how are"], + True, + {"max_gen_toks": 100, "temperature": 0.7, "until": ["hi"]}, + { + "prompt": "Hello, how are", + "model": "gpt-3.5-turbo", + "max_tokens": 100, + "temperature": 0.7, + "stop": ["hi"], + "seed": 1234, + }, + ), + ( + ["Hello, how are", "you"], + True, + {}, + { + "prompt": "Hello, how are", + "model": "gpt-3.5-turbo", + "max_tokens": 256, + "temperature": 0, + "stop": [], + "seed": 1234, + }, + ), + ], +) +def test_model_generate_call_usage( + api, input_messages, generate, gen_kwargs, expected_payload +): + with patch("requests.post") as mock_post: + mock_response = MagicMock() + mock_response.json.return_value = {"result": "success"} + mock_post.return_value = mock_response + + # Act + result = api.model_call( + input_messages, generate=generate, gen_kwargs=gen_kwargs + ) + + # Assert + mock_post.assert_called_once() + _, kwargs = mock_post.call_args + assert "json" in kwargs + assert kwargs["json"] == expected_payload + assert result == {"result": "success"} + + +@pytest.mark.parametrize( + "input_messages, generate, gen_kwargs, expected_payload", + [ + ( + [[1, 2, 3, 4, 5]], + False, + None, + { + "model": "EleutherAI/pythia-1b", + "prompt": [[1, 2, 3, 4, 5]], + "max_tokens": 1, + "logprobs": 1, + "echo": True, + "seed": 1234, + "temperature": 0, + }, + ), + ], +) +def test_model_tokenized_call_usage( + api_tokenized, input_messages, generate, gen_kwargs, expected_payload +): + with patch("requests.post") as mock_post: + mock_response = MagicMock() + mock_response.json.return_value = {"result": "success"} + mock_post.return_value = mock_response + + # Act + result = api_tokenized.model_call( + input_messages, generate=generate, gen_kwargs=gen_kwargs + ) + + # Assert + mock_post.assert_called_once() + _, kwargs = mock_post.call_args + assert "json" in kwargs + assert kwargs["json"] == expected_payload + assert result == {"result": "success"} + + +class DummyAsyncContextManager: + def __init__(self, result): + self.result = result + + async def __aenter__(self): + return self.result + + async def __aexit__(self, exc_type, exc, tb): + pass + + +@pytest.mark.parametrize( + "expected_inputs, expected_ctxlens, expected_cache_keys", + [ + ( + [ + [1, 2, 3, 4, 5], + [6, 7, 8, 9, 10], + [11, 12, 13, 14, 15], + [16, 17, 18, 19, 20], + ], + [3, 3, 3, 3], + ["cache_key1", "cache_key2", "cache_key3", "cache_key4"], + ), + ], +) +def test_get_batched_requests_with_no_ssl( + api_batch_ssl_tokenized, expected_inputs, expected_ctxlens, expected_cache_keys +): + with ( + patch( + "lm_eval.models.api_models.TCPConnector", autospec=True + ) as mock_connector, + patch( + "lm_eval.models.api_models.ClientSession", autospec=True + ) as mock_client_session, + patch( + "lm_eval.models.openai_completions.LocalCompletionsAPI.parse_logprobs", + autospec=True, + ) as mock_parse, + ): + mock_session_instance = AsyncMock() + mock_post_response = AsyncMock() + mock_post_response.status = 200 + mock_post_response.ok = True + mock_post_response.json = AsyncMock(return_value={"mocked": "response"}) + mock_post_response.raise_for_status = lambda: None + mock_session_instance.post = lambda *args, **kwargs: DummyAsyncContextManager( + mock_post_response + ) + mock_client_session.return_value.__aenter__.return_value = mock_session_instance + mock_parse.return_value = [(1.23, True), (4.56, False)] + + async def run(): + return await api_batch_ssl_tokenized.get_batched_requests( + expected_inputs, + expected_cache_keys, + generate=False, + ctxlens=expected_ctxlens, + ) + + result_batches = asyncio.run(run()) + + mock_connector.assert_called_with(limit=2, ssl=False) + assert result_batches diff --git a/lm-evaluation-harness/tests/models/test_gguf.py b/lm-evaluation-harness/tests/models/test_gguf.py new file mode 100644 index 0000000000000000000000000000000000000000..b5e197e77418fe83e3cc1cf96e23223b80afe633 --- /dev/null +++ b/lm-evaluation-harness/tests/models/test_gguf.py @@ -0,0 +1,152 @@ +import hashlib +import json +import os +import pickle +import unittest +from unittest.mock import patch + +from lm_eval.api.instance import Instance +from lm_eval.models.gguf import GGUFLM + + +base_url = "https://matthoffner-ggml-llm-api.hf.space" + + +def gguf_completion_mock(base_url=None, **kwargs): + # Generate a hash from the parameters + hash_kwargs = {"base_url": base_url, **kwargs} + parameters_hash = hashlib.sha256( + json.dumps(hash_kwargs, sort_keys=True).encode("utf-8") + ).hexdigest() + + fname = f"./tests/testdata/gguf_test_{parameters_hash}.pkl" + + if os.path.exists(fname): + with open(fname, "rb") as fh: + return pickle.load(fh) + else: + print("The file does not exist, attempting to write...") + if "stop" in kwargs: + result = { + "choices": [ + { + "text": f"generated text until {kwargs['stop']}", + "logprobs": {"token_logprobs": [-1.2345], "text_offset": 0}, + "finish_reason": "length", + } + ] + } + else: + # generated with # curl -X 'POST' 'http://localhost:8000/v1/completions' -H 'accept: application/json' -H 'Content-Type: application/json' -d '{"prompt": "string", "logprobs": 10, "temperature": 0.0, "max_tokens": 1, "echo": true}' + result = { + "id": "cmpl-4023976b-bc6a-43b0-a5a9-629f4216c7f3", + "object": "text_completion", + "created": 1700511361, + "model": "../llama-2-7b.Q8_0.gguf", + "choices": [ + { + "text": "string(", + "index": 0, + "logprobs": { + "text_offset": [0, 7], + "token_logprobs": [None, -1.033263319857306], + "tokens": [" string", "("], + "top_logprobs": [ + None, + { + "(": -1.033263319857306, + "[]": -2.6530743779017394, + ".": -3.0377145947291324, + "\n": -3.0399156750513976, + "_": -3.510376089937872, + " =": -3.6957918347193663, + ",": -3.9309459866358702, + " of": -4.2834550083949035, + '("': -4.322762841112799, + "()": -4.426229113466925, + }, + ], + }, + "finish_reason": "length", + } + ], + "usage": { + "prompt_tokens": 2, + "completion_tokens": 1, + "total_tokens": 3, + }, + } + + try: + os.makedirs(os.path.dirname(fname), exist_ok=True) + print("Writing file at", fname) + with open(fname, "wb") as fh: + pickle.dump(result, fh) + print("File written successfully") + except Exception as e: + print("File writing failed:", e) + + return result + + +class GGUFLMTest(unittest.TestCase): + @patch( + "lm_eval.models.gguf.GGUFLM.gguf_completion", side_effect=gguf_completion_mock + ) + def test_loglikelihood(self, gguf_completion_mock): + lm = GGUFLM(base_url) + + # Test loglikelihood + requests = [ + Instance( + request_type="loglikelihood", + doc=args, + arguments=args, + idx=i, + ) + for i, args in enumerate([("str", "ing"), ("str", "ing")]) + ] + res = lm.loglikelihood(requests) + + # Assert the loglikelihood response is correct + expected_res = [(logprob, True) for logprob in [0, 0]] + self.assertEqual(res, expected_res) + + @patch( + "lm_eval.models.gguf.GGUFLM.gguf_completion", side_effect=gguf_completion_mock + ) + def test_generate_until(self, gguf_completion_mock): + lm = GGUFLM(base_url) + + # Test generate_until + requests = [ + Instance( + request_type="generate_until", + doc={"input": doc}, + arguments=(doc, {"until": stop}), + idx=i, + ) + for i, (doc, stop) in enumerate([("input1", "stop1"), ("input2", "stop2")]) + ] + + res = lm.generate_until(requests) + + # Assert the generate_until response is correct + expected_res = ["generated text until stop1", "generated text until stop2"] + self.assertEqual(res, expected_res) + + # @patch('lm_eval.models.gguf.GGUFLM.gguf_completion', side_effect=gguf_completion_mock) + # def test_loglikelihood_rolling(self, gguf_completion_mock): + # lm = GGUFLM(base_url) + + # # Test loglikelihood_rolling + # requests = ["input1", "input2"] + # res = lm.loglikelihood_rolling(requests) + + # # Assert the loglikelihood_rolling response is correct + # expected_res = [(-1.2345, True), (-1.2345, True)] + # self.assertEqual(res, expected_res) + + +if __name__ == "__main__": + unittest.main() diff --git a/lm-evaluation-harness/tests/models/test_gptqmodel.py b/lm-evaluation-harness/tests/models/test_gptqmodel.py new file mode 100644 index 0000000000000000000000000000000000000000..9e3831c8fbacd96cae75f462fd14c57b048acd77 --- /dev/null +++ b/lm-evaluation-harness/tests/models/test_gptqmodel.py @@ -0,0 +1,54 @@ +from typing import List + +import pytest + +import lm_eval + + +def assert_less_than(value, threshold, desc): + if value is not None: + assert float(value) < threshold, f"{desc} should be less than {threshold}" + + +@pytest.mark.skip(reason="requires CUDA") +class Test_GPTQModel: + gptqmodel = pytest.importorskip("gptqmodel", minversion="1.0.9") + MODEL_ID = "ModelCloud/Opt-125-GPTQ-4bit-10-25-2024" + + def test_gptqmodel(self) -> None: + acc = "acc" + acc_norm = "acc_norm" + acc_value = None + acc_norm_value = None + task = "arc_easy" + + model_args = f"pretrained={self.MODEL_ID},gptqmodel=True" + + tasks: List[str] = [task] + + results = lm_eval.simple_evaluate( + model="hf", + model_args=model_args, + tasks=tasks, + device="cuda", + ) + + column = "results" + dic = results.get(column, {}).get(self.task) + if dic is not None: + if "alias" in dic: + _ = dic.pop("alias") + items = sorted(dic.items()) + for k, v in items: + m, _, f = k.partition(",") + if m.endswith("_stderr"): + continue + + if m == acc: + acc_value = "%.4f" % v if isinstance(v, float) else v + + if m == acc_norm: + acc_norm_value = "%.4f" % v if isinstance(v, float) else v + + assert_less_than(acc_value, 0.43, "acc") + assert_less_than(acc_norm_value, 0.39, "acc_norm") diff --git a/lm-evaluation-harness/tests/models/test_hf_steered.py b/lm-evaluation-harness/tests/models/test_hf_steered.py new file mode 100644 index 0000000000000000000000000000000000000000..efa0cc03392cadb379aa22a2f44deb4dd950167d --- /dev/null +++ b/lm-evaluation-harness/tests/models/test_hf_steered.py @@ -0,0 +1,169 @@ +# ruff: noqa +from __future__ import annotations + +import os +import sys +from pathlib import Path + +import numpy as np +import pytest +import torch + +from lm_eval import tasks +from lm_eval.api.instance import Instance + +pytest.skip("dependency conflict on CI", allow_module_level=True) + +os.environ["TOKENIZERS_PARALLELISM"] = "false" +task_manager = tasks.TaskManager() + +TEST_STRING = "foo bar" + + +class Test_SteeredModel: + from lm_eval.models.hf_steered import SteeredModel + + torch.use_deterministic_algorithms(True) + task_list = task_manager.load_task_or_group(["arc_easy", "gsm8k", "wikitext"]) + version_minor = sys.version_info.minor + multiple_choice_task = task_list["arc_easy"] # type: ignore + multiple_choice_task.build_all_requests(limit=10, rank=0, world_size=1) + MULTIPLE_CH: list[Instance] = multiple_choice_task.instances + generate_until_task = task_list["gsm8k"] # type: ignore + generate_until_task._config.generation_kwargs["max_gen_toks"] = 10 + generate_until_task.set_fewshot_seed(1234) # fewshot random generator seed + generate_until_task.build_all_requests(limit=10, rank=0, world_size=1) + generate_until: list[Instance] = generate_until_task.instances + rolling_task = task_list["wikitext"] # type: ignore + rolling_task.build_all_requests(limit=10, rank=0, world_size=1) + ROLLING: list[Instance] = rolling_task.instances + + MULTIPLE_CH_RES = [ + -41.79737854003906, + -42.964412689208984, + -33.909732818603516, + -37.055198669433594, + -22.980390548706055, + -20.268718719482422, + -14.76205062866211, + -27.887500762939453, + -15.797225952148438, + -15.914306640625, + -13.01901626586914, + -18.053699493408203, + -13.33236312866211, + -13.35921859741211, + -12.12301254272461, + -11.86703109741211, + -47.02234649658203, + -47.69982147216797, + -36.420310974121094, + -50.065345764160156, + -16.742475509643555, + -18.542402267456055, + -26.460208892822266, + -20.307228088378906, + -17.686725616455078, + -21.752883911132812, + -33.17183303833008, + -39.21712112426758, + -14.78198528289795, + -16.775150299072266, + -11.49817180633545, + -15.404842376708984, + -13.141255378723145, + -15.870940208435059, + -15.29050064086914, + -12.36030387878418, + -44.557891845703125, + -55.43851089477539, + -52.66646194458008, + -56.289222717285156, + ] + generate_until_RES = [ + " The average of $2.50 each is $", + " A robe takes 2 bolts of blue fiber and half", + " $50,000 in repairs.\n\nQuestion", + " He runs 1 sprint 3 times a week.", + " They feed each of her chickens three cups of mixed", + " The price of the glasses is $5, but", + " The total percentage of students who said they like to", + " Carla is downloading a 200 GB file. Normally", + " John drives for 3 hours at a speed of 60", + " Eliza sells 4 tickets to 5 friends so she", + ] + ROLLING_RES = [ + -3604.61328125, + -19778.67626953125, + -8835.119384765625, + -27963.37841796875, + -7636.4351806640625, + -9491.43603515625, + -41047.35205078125, + -8396.804443359375, + -45966.24645996094, + -7159.05322265625, + ] + LM = SteeredModel( + pretrained="EleutherAI/pythia-70m", + device="cpu", + dtype="float32", + steer_path="tests/testconfigs/sparsify_intervention.csv", + ) + + def test_load_with_sae_lens(self) -> None: + from lm_eval.models.hf_steered import SteeredModel + + SteeredModel( + pretrained="EleutherAI/pythia-70m", + device="cpu", + dtype="float32", + steer_path="tests/testconfigs/sae_lens_intervention.csv", + ) + + assert True + + def test_loglikelihood(self) -> None: + res = self.LM.loglikelihood(self.MULTIPLE_CH) + _RES, _res = self.MULTIPLE_CH_RES, [r[0] for r in res] + # log samples to CI + dir_path = Path("test_logs") + dir_path.mkdir(parents=True, exist_ok=True) + + file_path = dir_path / f"outputs_log_{self.version_minor}.txt" + file_path = file_path.resolve() + with open(file_path, "w", encoding="utf-8") as f: + f.write("\n".join(str(x) for x in _res)) + assert np.allclose(_res, _RES, atol=1e-2) + # check indices for Multiple Choice + argmax_RES, argmax_res = ( + np.argmax(np.array(_RES).reshape(-1, 4), axis=1), + np.argmax(np.array(_res).reshape(-1, 4), axis=1), + ) + assert (argmax_RES == argmax_res).all() + + def test_generate_until(self) -> None: + res = self.LM.generate_until(self.generate_until) + assert res == self.generate_until_RES + + def test_loglikelihood_rolling(self) -> None: + res = self.LM.loglikelihood_rolling(self.ROLLING) + assert np.allclose(res, self.ROLLING_RES, atol=1e-1) + + def test_toc_encode(self) -> None: + res = self.LM.tok_encode(TEST_STRING) + assert res == [12110, 2534] + + def test_toc_decode(self) -> None: + res = self.LM.tok_decode([12110, 2534]) + assert res == TEST_STRING + + def test_batch_encode(self) -> None: + res = self.LM.tok_batch_encode([TEST_STRING, "bar foo"])[0].tolist() + assert res == [[12110, 2534], [2009, 17374]] + + def test_model_generate(self) -> None: + context = self.LM.tok_batch_encode([TEST_STRING])[0] + res = self.LM._model_generate(context, max_length=10, stop=["\n\n"]) + res = self.LM.tok_decode(res[0]) + assert res == "foo bar\n !info bar" diff --git a/lm-evaluation-harness/tests/models/test_huggingface.py b/lm-evaluation-harness/tests/models/test_huggingface.py new file mode 100644 index 0000000000000000000000000000000000000000..442696928c065551bf2ff781420b9459359d5ddf --- /dev/null +++ b/lm-evaluation-harness/tests/models/test_huggingface.py @@ -0,0 +1,153 @@ +from __future__ import annotations + +import os +import sys +from pathlib import Path + +import numpy as np +import tokenizers +import torch +from packaging.version import parse as parse_version + +from lm_eval import tasks +from lm_eval.api.instance import Instance +from lm_eval.models.huggingface import HFLM + + +os.environ["TOKENIZERS_PARALLELISM"] = "false" +task_manager = tasks.TaskManager() + +TEST_STRING = "foo bar" + + +class Test_HFLM: + torch.use_deterministic_algorithms(True) + task_list = task_manager.load_task_or_group(["arc_easy", "gsm8k", "wikitext"]) + version_minor = sys.version_info.minor + multiple_choice_task = task_list["arc_easy"] # type: ignore + multiple_choice_task.build_all_requests(limit=10, rank=0, world_size=1) + MULTIPLE_CH: list[Instance] = multiple_choice_task.instances + generate_until_task = task_list["gsm8k"] # type: ignore + generate_until_task._config.generation_kwargs["max_gen_toks"] = 10 + generate_until_task.set_fewshot_seed(1234) # fewshot random generator seed + generate_until_task.build_all_requests(limit=10, rank=0, world_size=1) + generate_until: list[Instance] = generate_until_task.instances + rolling_task = task_list["wikitext"] # type: ignore + rolling_task.build_all_requests(limit=10, rank=0, world_size=1) + ROLLING: list[Instance] = rolling_task.instances + + MULTIPLE_CH_RES = [ + -41.902435302734375, + -42.939308166503906, + -33.914180755615234, + -37.07139205932617, + -22.95258331298828, + -20.342208862304688, + -14.818366050720215, + -27.942853927612305, + -15.80704116821289, + -15.936427116394043, + -13.052018165588379, + -18.04828453063965, + -13.345029830932617, + -13.366025924682617, + -12.127134323120117, + -11.872495651245117, + -47.10598373413086, + -47.76410675048828, + -36.4406852722168, + -50.0289421081543, + -16.72093963623047, + -18.535587310791016, + -26.46993637084961, + -20.355995178222656, + -17.757919311523438, + -21.80595588684082, + -33.1990852355957, + -39.28636932373047, + -14.759679794311523, + -16.753942489624023, + -11.486852645874023, + -15.42177677154541, + -13.15798282623291, + -15.887393951416016, + -15.28614616394043, + -12.339089393615723, + -44.59441375732422, + -55.40888214111328, + -52.70050811767578, + -56.25089645385742, + ] + generate_until_RES = [ + " The average of $2.50 each is $", + " A robe takes 2 bolts of blue fiber and half", + " $50,000 in repairs.\n\nQuestion", + " He runs 1 sprint 3 times a week.", + " They feed each of her chickens three cups of mixed", + " The price of the glasses is $5, but", + " The total percentage of students who said they like to", + " Carla is downloading a 200 GB file. Normally", + " John drives for 3 hours at a speed of 60", + " Eliza sells 4 tickets to 5 friends so she", + ] + ROLLING_RES = [ + -3603.6328125, + -19779.23974609375, + -8834.16455078125, + -27967.591796875, + -7636.794982910156, + -9491.93505859375, + -41043.4248046875, + -8397.689819335938, + -45969.47155761719, + -7158.90625, + ] + LM = HFLM(pretrained="EleutherAI/pythia-70m", device="cpu", dtype="float32") + + def test_logliklihood(self) -> None: + res = self.LM.loglikelihood(self.MULTIPLE_CH) + _RES, _res = self.MULTIPLE_CH_RES, [r[0] for r in res] + # log samples to CI + dir_path = Path("test_logs") + dir_path.mkdir(parents=True, exist_ok=True) + + file_path = dir_path / f"outputs_log_{self.version_minor}.txt" + file_path = file_path.resolve() + with open(file_path, "w", encoding="utf-8") as f: + f.write("\n".join(str(x) for x in _res)) + assert np.allclose(_res, _RES, atol=1e-2) + # check indices for Multiple Choice + argmax_RES, argmax_res = ( + np.argmax(np.array(_RES).reshape(-1, 4), axis=1), + np.argmax(np.array(_res).reshape(-1, 4), axis=1), + ) + assert (argmax_RES == argmax_res).all() + + def test_generate_until(self) -> None: + res = self.LM.generate_until(self.generate_until) + assert res == self.generate_until_RES + + def test_logliklihood_rolling(self) -> None: + res = self.LM.loglikelihood_rolling(self.ROLLING) + assert np.allclose(res, self.ROLLING_RES, atol=1e-1) + + def test_toc_encode(self) -> None: + res = self.LM.tok_encode(TEST_STRING) + assert res == [12110, 2534] + + def test_toc_decode(self) -> None: + res = self.LM.tok_decode([12110, 2534]) + assert res == TEST_STRING + + def test_batch_encode(self) -> None: + res = self.LM.tok_batch_encode([TEST_STRING, "bar foo"])[0].tolist() + assert res == [[12110, 2534], [2009, 17374]] + + def test_model_generate(self) -> None: + context = self.LM.tok_batch_encode([TEST_STRING])[0] + res = self.LM._model_generate(context, max_length=10, stop=["\n\n"]) + res = self.LM.tok_decode(res[0]) + if parse_version(tokenizers.__version__) >= parse_version("0.20.0"): + assert res == "foo bar\n !info bar" + else: + assert res == "foo bar\n!info bar" diff --git a/lm-evaluation-harness/tests/models/test_neuralmagic.py b/lm-evaluation-harness/tests/models/test_neuralmagic.py new file mode 100644 index 0000000000000000000000000000000000000000..e0a36ceeb2847f60449572777a5b3a9595ccafd4 --- /dev/null +++ b/lm-evaluation-harness/tests/models/test_neuralmagic.py @@ -0,0 +1,62 @@ +import pytest + +from lm_eval import evaluator +from lm_eval.api.registry import get_model + + +SPARSEML_MODELS_TASKS = [ + # loglikelihood + ("facebook/opt-125m", "lambada_openai"), + # loglikelihood_rolling + ("hf-internal-testing/tiny-random-gpt2", "wikitext"), + # generate_until + ("mgoin/tiny-random-llama-2-quant", "gsm8k"), +] + +DEEPSPARSE_MODELS_TASKS = [ + # loglikelihood + ("hf:mgoin/llama2.c-stories15M-quant-ds", "lambada_openai"), + # loglikelihood_rolling (not supported yet) + # ("hf:mgoin/llama2.c-stories15M-quant-ds", "wikitext"), + # generate_until + ("hf:mgoin/llama2.c-stories15M-quant-ds", "gsm8k"), +] + + +@pytest.mark.skip(reason="test failing") +@pytest.mark.parametrize("model_id,task", SPARSEML_MODELS_TASKS) +def test_sparseml_eval(model_id, task): + lm = get_model("sparseml").create_from_arg_string( + f"pretrained={model_id}", + { + "batch_size": 1, + "device": "cpu", + "dtype": "float32", + }, + ) + + limit = 5 + evaluator.simple_evaluate( + model=lm, + tasks=[task], + num_fewshot=0, + limit=limit, + ) + + +@pytest.mark.parametrize("model_id,task", DEEPSPARSE_MODELS_TASKS) +def test_deepsparse_eval(model_id, task): + lm = get_model("deepsparse").create_from_arg_string( + f"pretrained={model_id}", + { + "batch_size": 1, + }, + ) + + limit = 5 + evaluator.simple_evaluate( + model=lm, + tasks=[task], + num_fewshot=0, + limit=limit, + ) diff --git a/lm-evaluation-harness/tests/models/test_openvino.py b/lm-evaluation-harness/tests/models/test_openvino.py new file mode 100644 index 0000000000000000000000000000000000000000..b8f13cd9adb3d3850a28055c9a6daf43d40e3874 --- /dev/null +++ b/lm-evaluation-harness/tests/models/test_openvino.py @@ -0,0 +1,92 @@ +import random +import tempfile +from pathlib import Path + +import pytest +from optimum.intel import OVModelForCausalLM +from transformers import AutoTokenizer + +from lm_eval import evaluator +from lm_eval.api.registry import get_model + + +SUPPORTED_ARCHITECTURES_TASKS = { + "facebook/opt-125m": "lambada_openai", + "hf-internal-testing/tiny-random-gpt2": "wikitext", +} + + +@pytest.mark.parametrize("model_id,task", SUPPORTED_ARCHITECTURES_TASKS.items()) +def test_evaluator(model_id, task): + with tempfile.TemporaryDirectory() as tmpdirname: + model = OVModelForCausalLM.from_pretrained( + model_id, export=True, use_cache=True + ) + model.save_pretrained(tmpdirname) + tokenizer = AutoTokenizer.from_pretrained(model_id) + tokenizer.save_pretrained(tmpdirname) + + lm = get_model("openvino").create_from_arg_string( + f"pretrained={tmpdirname}", + { + "batch_size": 1, + "device": "cpu", + }, + ) + + def ll_fn(reqs): + for ctx, cont in [req.args for req in reqs]: + if len(ctx) == 0: + continue + # space convention + assert ctx[-1] != " " + assert cont[0] == " " or ctx[-1] == "\n" + + res = [] + + random.seed(42) + for _ in reqs: + res.extend([(-random.random(), False)]) + + return res + + def ll_perp_fn(reqs): + for (string,) in [req.args for req in reqs]: + assert isinstance(string, str) + + res = [] + random.seed(42) + for _ in reqs: + res.extend([-random.random()]) + + return res + + lm.loglikelihood = ll_fn + lm.loglikelihood_rolling = ll_perp_fn + + limit = 10 + evaluator.simple_evaluate( + model=lm, + tasks=[task], + num_fewshot=0, + limit=limit, + bootstrap_iters=10, + ) + + +def test_ov_config(): + """Test that if specified, a custom OpenVINO config is loaded correctly""" + model_id = "hf-internal-testing/tiny-random-gpt2" + with tempfile.TemporaryDirectory() as tmpdirname: + config_file = str(Path(tmpdirname) / "ov_config.json") + with open(Path(config_file), "w", encoding="utf-8") as f: + f.write('{"DYNAMIC_QUANTIZATION_GROUP_SIZE" : "32"}') + lm = get_model("openvino").create_from_arg_string( + f"pretrained={model_id},ov_config={config_file}" + ) + assert ( + lm.model.request.get_compiled_model().get_property( + "DYNAMIC_QUANTIZATION_GROUP_SIZE" + ) + == 32 + ) diff --git a/lm-evaluation-harness/tests/models/test_sglang.py b/lm-evaluation-harness/tests/models/test_sglang.py new file mode 100644 index 0000000000000000000000000000000000000000..e7ed33d6132cc640652c46408c68d7b6aab71a2d --- /dev/null +++ b/lm-evaluation-harness/tests/models/test_sglang.py @@ -0,0 +1,125 @@ +from typing import List + +import pytest +import torch + +from lm_eval import evaluate, simple_evaluate, tasks +from lm_eval.api.instance import Instance +from lm_eval.tasks import get_task_dict + + +task_manager = tasks.TaskManager() + + +# We refer to vLLM's test but modify the trigger condition. +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +# @pytest.mark.skip(reason="requires CUDA") +class Test_SGlang: + sglang = pytest.importorskip("sglang") + + task_list = task_manager.load_task_or_group(["arc_easy", "gsm8k", "wikitext"]) + multiple_choice_task = task_list["arc_easy"] # type: ignore + multiple_choice_task.build_all_requests(limit=10, rank=0, world_size=1) + MULTIPLE_CH: List[Instance] = multiple_choice_task.instances + generate_until_task = task_list["gsm8k"] # type: ignore + generate_until_task._config.generation_kwargs["max_gen_toks"] = 10 + generate_until_task.build_all_requests(limit=10, rank=0, world_size=1) + generate_until: List[Instance] = generate_until_task.instances + rolling_task = task_list["wikitext"] # type: ignore + rolling_task.build_all_requests(limit=10, rank=0, world_size=1) + ROLLING: List[Instance] = rolling_task.instances + + @classmethod + def setup_class(cls): + try: + from lm_eval.models.sglang_causallms import SGLangLM + + # NOTE(jinwei): EleutherAI/pythia-70m is not supported by SGlang yet. Instead we use Qwen models. + cls.LM = SGLangLM( + pretrained="Qwen/Qwen2-1.5B-Instruct", + batch_size=1, + tp_size=1, + max_model_len=1024, + ) + except Exception as e: + pytest.fail(f"🔥 SGLangLM failed to initialize: {e}") + + def test_logliklihood(self) -> None: + res = self.LM.loglikelihood(self.MULTIPLE_CH) + assert len(res) == len(self.MULTIPLE_CH) + for x in res: + assert isinstance(x[0], float) + + def test_generate_until(self) -> None: + res = self.LM.generate_until(self.generate_until) + assert len(res) == len(self.generate_until) + for x in res: + assert isinstance(x, str) + + # NOTE(Jinwei):A100 80GB is enough for our tests. If you run the last test "test_logliklihood_rolling" and OOM happens, please reduce the "max_model_len". + def test_logliklihood_rolling(self) -> None: + res = self.LM.loglikelihood_rolling(self.ROLLING) + for x in res: + assert isinstance(x, float) + + # def test_simple_evaluate(self)-> None: + # results = simple_evaluate( + # model =self.LM, + # tasks=["arc_easy"], + # # num_fewshot=0, + # task_manager=task_manager, + # limit= 10, + # ) + # print(results) + # accuracy = results["results"]["arc_easy"]["acc,none"] + # print(f"Accuracy: {accuracy}") + + # def test_evaluate(self)-> None: + # tasks=["arc_easy"] + # task_dict = get_task_dict(tasks, task_manager) + # results = evaluate( + # lm=self.LM, + # task_dict=task_dict, + # limit= 10, + # ) + # print(results) + # accuracy = results["results"]["arc_easy"]["acc,none"] + # print(f"Accuracy: {accuracy}") + + # TODO(jinwei): find out the outpt differences for "gsm_8k" with simple_evalute() and evaluate(). There are some errors in parser as well. + def test_evaluator(self) -> None: + simple_results = simple_evaluate( + model=self.LM, + tasks=["arc_easy"], + task_manager=task_manager, + limit=10, + ) + assert simple_results is not None, "simple_evaluate returned None" + # The accuracy for 10 data points is 0.7. Setting up a threshold of 0.5 provides a buffer to account for these fluctuations. + assert simple_results["results"]["arc_easy"]["acc,none"] >= 0.5, ( + "The accuracy for simple_evaluate() is below 0.5!" + ) + task_dict = get_task_dict(["arc_easy"], task_manager) + evaluate_results = evaluate( + lm=self.LM, + task_dict=task_dict, + limit=10, + ) + assert evaluate_results is not None, "evaluate returned None" + # The accuracy for 10 data points is 0.7. Setting up a threshold of 0.5 provides a buffer to account for these fluctuations. + assert evaluate_results["results"]["arc_easy"]["acc,none"] >= 0.5, ( + "The accuracy for evaluate() is below 0.5!" + ) + + assert set(simple_results["results"].keys()) == set( + evaluate_results["results"].keys() + ), "Mismatch in task keys between simple_evaluate and evaluate" + + for task in simple_results["results"]: + assert ( + simple_results["results"][task] == evaluate_results["results"][task] + ), f"Mismatch in results for {task}" + + print( + "✅ test_evaluator passed: simple_evaluate and evaluate results are identical." + ) diff --git a/lm-evaluation-harness/tests/models/test_vllm.py b/lm-evaluation-harness/tests/models/test_vllm.py new file mode 100644 index 0000000000000000000000000000000000000000..01363bc8dc31b43549f62120a8ce9fde0788b144 --- /dev/null +++ b/lm-evaluation-harness/tests/models/test_vllm.py @@ -0,0 +1,50 @@ +from typing import List + +import pytest + +from lm_eval import tasks +from lm_eval.api.instance import Instance + + +task_manager = tasks.TaskManager() + + +@pytest.mark.skip(reason="requires CUDA") +class Test_VLLM: + vllm = pytest.importorskip("vllm") + try: + from lm_eval.models.vllm_causallms import VLLM + + LM = VLLM(pretrained="EleutherAI/pythia-70m") + except ModuleNotFoundError: + pass + # torch.use_deterministic_algorithms(True) + task_list = task_manager.load_task_or_group(["arc_easy", "gsm8k", "wikitext"]) + multiple_choice_task = task_list["arc_easy"] # type: ignore + multiple_choice_task.build_all_requests(limit=10, rank=0, world_size=1) + MULTIPLE_CH: List[Instance] = multiple_choice_task.instances + generate_until_task = task_list["gsm8k"] # type: ignore + generate_until_task._config.generation_kwargs["max_gen_toks"] = 10 + generate_until_task.build_all_requests(limit=10, rank=0, world_size=1) + generate_until: List[Instance] = generate_until_task.instances + rolling_task = task_list["wikitext"] # type: ignore + rolling_task.build_all_requests(limit=10, rank=0, world_size=1) + ROLLING: List[Instance] = rolling_task.instances + + # TODO: make proper tests + def test_logliklihood(self) -> None: + res = self.LM.loglikelihood(self.MULTIPLE_CH) + assert len(res) == len(self.MULTIPLE_CH) + for x in res: + assert isinstance(x[0], float) + + def test_generate_until(self) -> None: + res = self.LM.generate_until(self.generate_until) + assert len(res) == len(self.generate_until) + for x in res: + assert isinstance(x, str) + + def test_logliklihood_rolling(self) -> None: + res = self.LM.loglikelihood_rolling(self.ROLLING) + for x in res: + assert isinstance(x, float) diff --git a/lm-evaluation-harness/tests/test_cli.py b/lm-evaluation-harness/tests/test_cli.py new file mode 100644 index 0000000000000000000000000000000000000000..feaa7340d6a36da7c98f3c44128702fc33f0e770 --- /dev/null +++ b/lm-evaluation-harness/tests/test_cli.py @@ -0,0 +1,43 @@ +import argparse + +import pytest + +import lm_eval.__main__ + + +def test_cli_parse_error(): + """ + Assert error raised if cli args argument doesn't have type + """ + with pytest.raises(ValueError): + parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument( + "--model", "-m", type=str, default="hf", help="Name of model e.g. `hf`" + ) + parser.add_argument( + "--tasks", + "-t", + default=None, + metavar="task1,task2", + help="To get full list of tasks, use the command lm-eval --tasks list", + ) + lm_eval.__main__.check_argument_types(parser) + + +def test_cli_parse_no_error(): + """ + Assert typed arguments are parsed correctly + """ + parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument( + "--model", "-m", type=str, default="hf", help="Name of model e.g. `hf`" + ) + parser.add_argument( + "--tasks", + "-t", + type=str, + default=None, + metavar="task1,task2", + help="To get full list of tasks, use the command lm-eval --tasks list", + ) + lm_eval.__main__.check_argument_types(parser) diff --git a/lm-evaluation-harness/tests/test_evaluator.py b/lm-evaluation-harness/tests/test_evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..cc5f70fa73edf6a9dbd4d8d325dc3cc868eace6c --- /dev/null +++ b/lm-evaluation-harness/tests/test_evaluator.py @@ -0,0 +1,151 @@ +import os +import re +from typing import List + +import pytest + +import lm_eval.api as api +import lm_eval.evaluator as evaluator +from lm_eval import tasks +from lm_eval.utils import make_table + + +os.environ["TOKENIZERS_PARALLELISM"] = "false" +# TODO: more fine grained unit tests rather than this big honking integration +# test once we break evaluator into smaller, more manageable pieces + + +@pytest.mark.parametrize( + "task_name,limit,model,model_args,bootstrap_iters", + [ + ( + ["arc_easy"], + 10, + "hf", + "pretrained=EleutherAI/pythia-160m,dtype=float32,device=cpu", + 0, + ), + ( + ["mmlu_abstract_algebra"], + None, + "hf", + "pretrained=EleutherAI/pythia-160m,dtype=float32,device=cpu", + 10000, + ), + ], + ids=lambda d: f"{d}", +) +def test_evaluator( + task_name: List[str], limit: int, model: str, model_args: str, bootstrap_iters: int +): + e1 = evaluator.simple_evaluate( + model=model, + tasks=task_name, + limit=limit, + model_args=model_args, + bootstrap_iters=bootstrap_iters, + ) + assert e1 is not None + + lm = api.registry.get_model(model).create_from_arg_string( + model_args, + { + "batch_size": None, + "max_batch_size": None, + "device": None, + }, + ) + task_manager = tasks.TaskManager() + task_dict = tasks.get_task_dict(task_name, task_manager) + + e2 = evaluator.evaluate( + lm=lm, + task_dict=task_dict, + limit=limit, + bootstrap_iters=bootstrap_iters, + ) + + assert e2 is not None + # check that caching is working + + def r(x): + if "arc_easy" in x["results"]: + return x["results"]["arc_easy"] + else: + return x["results"]["mmlu_abstract_algebra"] + + assert all( + x == y + for x, y in zip([y for _, y in r(e1).items()], [y for _, y in r(e2).items()]) + ) + + +@pytest.mark.parametrize( + "task_name,limit,model,model_args", + [ + ( + ["ai2_arc"], + 10, + "hf", + "pretrained=EleutherAI/pythia-14m,dtype=float32,device=cpu", + ), + ( + ["mmlu_stem"], + 10, + "hf", + "pretrained=EleutherAI/pythia-14m,dtype=float32,device=cpu", + ), + ( + ["lambada_openai"], + 10, + "hf", + "pretrained=EleutherAI/pythia-14m,dtype=float32,device=cpu", + ), + ( + ["wikitext"], + 10, + "hf", + "pretrained=EleutherAI/pythia-14m,dtype=float32,device=cpu", + ), + ], + ids=lambda d: f"{d}", +) +def test_printed_results(task_name: List[str], limit: int, model: str, model_args: str): + results = evaluator.simple_evaluate( + model=model, + tasks=task_name, + limit=limit, + model_args=model_args, + bootstrap_iters=0, + random_seed=0, + numpy_random_seed=0, + torch_random_seed=0, + fewshot_random_seed=0, + ) + + filename = "_".join( + ( + "-".join(task_name), + str(limit), + str(model), + re.sub(r"[^a-zA-Z0-9_\-\.]", "-", model_args), + ) + ) + filepath = f"./tests/testdata/{filename}.txt" + with open(filepath, "r") as f: + t1 = f.read().strip() + + t2 = make_table(results).strip() + + t1_lines, t2_lines = t1.splitlines(), t2.splitlines() + assert len(t1_lines) == len(t2_lines) + for t1_line, t2_line in zip(t1_lines, t2_lines): + t1_items, t2_items = t1_line.split("|"), t2_line.split("|") + assert len(t1_items) == len(t2_items) + for t1_item, t2_item in zip(t1_items, t2_items): + try: + t1_item = float(t1_item) + t2_item = float(t2_item) + assert abs(t1_item - t2_item) < 0.3 + except ValueError: + assert t1_item == t2_item diff --git a/lm-evaluation-harness/tests/test_include_path.py b/lm-evaluation-harness/tests/test_include_path.py new file mode 100644 index 0000000000000000000000000000000000000000..debbdaf46436a74155542b91ea7762bf8c63cd3d --- /dev/null +++ b/lm-evaluation-harness/tests/test_include_path.py @@ -0,0 +1,93 @@ +import os + +import pytest + +import lm_eval.api as api +import lm_eval.evaluator as evaluator +from lm_eval import tasks + + +@pytest.mark.parametrize( + "limit,model,model_args", + [ + ( + 10, + "hf", + "pretrained=EleutherAI/pythia-160m,dtype=float32,device=cpu", + ), + ], +) +def test_include_correctness(limit: int, model: str, model_args: str): + task_name = ["arc_easy"] + + task_manager = tasks.TaskManager() + task_dict = tasks.get_task_dict(task_name, task_manager) + + e1 = evaluator.simple_evaluate( + model=model, + tasks=task_name, + limit=limit, + model_args=model_args, + ) + assert e1 is not None + + # run with evaluate() and "arc_easy" test config (included from ./testconfigs path) + lm = api.registry.get_model(model).create_from_arg_string( + model_args, + { + "batch_size": None, + "max_batch_size": None, + "device": None, + }, + ) + + task_name = ["arc_easy"] + + task_manager = tasks.TaskManager( + include_path=os.path.dirname(os.path.abspath(__file__)) + "/testconfigs", + include_defaults=False, + ) + task_dict = tasks.get_task_dict(task_name, task_manager) + + e2 = evaluator.evaluate( + lm=lm, + task_dict=task_dict, + limit=limit, + ) + + assert e2 is not None + # check that caching is working + + def r(x): + return x["results"]["arc_easy"] + + assert all( + x == y + for x, y in zip([y for _, y in r(e1).items()], [y for _, y in r(e2).items()]) + ) + + +# test that setting include_defaults = False works as expected and that include_path works +def test_no_include_defaults(): + task_name = ["arc_easy"] + + task_manager = tasks.TaskManager( + include_path=os.path.dirname(os.path.abspath(__file__)) + "/testconfigs", + include_defaults=False, + ) + # should succeed, because we've included an 'arc_easy' task from this dir + task_dict = tasks.get_task_dict(task_name, task_manager) + + # should fail, since ./testconfigs has no arc_challenge task + task_name = ["arc_challenge"] + with pytest.raises(KeyError): + task_dict = tasks.get_task_dict(task_name, task_manager) # noqa: F841 + + +# test that include_path containing a task shadowing another task's name fails +# def test_shadowed_name_fails(): + +# task_name = ["arc_easy"] + +# task_manager = tasks.TaskManager(include_path=os.path.dirname(os.path.abspath(__file__)) + "/testconfigs") +# task_dict = tasks.get_task_dict(task_name, task_manager) diff --git a/lm-evaluation-harness/tests/test_janitor.py b/lm-evaluation-harness/tests/test_janitor.py new file mode 100644 index 0000000000000000000000000000000000000000..e0ed7525a72fe938d80e03f1b9078ca454915ac2 --- /dev/null +++ b/lm-evaluation-harness/tests/test_janitor.py @@ -0,0 +1,446 @@ +import os +from collections import defaultdict + +from lm_eval.decontamination.janitor import ( + Janitor, + form_ngrams, + split_indices, + word_ngrams, + word_ngrams_indices, +) + + +os.environ["TOKENIZERS_PARALLELISM"] = "false" +TEST_SEQUENCE = ( + "Hello my name is Bob, I like eating pizza, chicken, chips and ice cream. Maybe I should eat some" + " more salad but it's so booooring. I just... like eating pizza, chicken, chips and ice cream so much." +) + +JANITOR_EXPECTED = ( + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing " + " characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " +) + +JANITOR_FILTH1 = "filth lots of dirty filthy filth" +JANITOR_FILTH2 = "filth lots of filthy dirty filth" + + +def simple_ngram(sequence, n): + ngrams = list() + ngram = [] + for x in sequence: + ngram.extend([x]) + if len(ngram) == n: + ngrams.extend([tuple(ngram)]) + ngram = ngram[1:] + + return ngrams + + +def test_form_ngrams(): + sequence = TEST_SEQUENCE + + n_values = [1, 2, 3, 5, 13] + for n in n_values: + comparison = simple_ngram(sequence, n) + result_to_test = list(form_ngrams(iter(sequence), n)) + assert len(comparison) == len(result_to_test) + assert comparison == result_to_test + + +def test_word_ngrams(): + sequence = TEST_SEQUENCE + + words = sequence.split() + + n_values = [1, 2, 3, 5, 13] + for n in n_values: + comparison = simple_ngram(words, n) + comparison = [" ".join(ngram) for ngram in comparison] + result_to_test = list(word_ngrams(sequence, n)) + assert len(comparison) == len(result_to_test) + assert result_to_test == comparison + + +def test_split_indices(): + sequence = TEST_SEQUENCE + + comparison = [] + current_word = "" + for i, c in enumerate(sequence): + if c != " ": + current_word += c + else: + if current_word: + comparison.extend([(current_word, (i - len(current_word), i - 1))]) + current_word = "" + + if current_word: + len_sequence = len(sequence) + comparison.extend( + [ + ( + current_word, + (len_sequence - len(current_word), len_sequence - 1), + ) + ] + ) + current_word = "" + + result_to_test = list(split_indices(sequence)) + assert len(comparison) == len(result_to_test) + assert comparison == result_to_test + + +def test_word_ngrams_indices(): + sequence = TEST_SEQUENCE + + n_values = [1, 2, 3, 5, 13] + + for n in n_values: + ngrams = [" ".join(ngram) for ngram in simple_ngram(sequence.split(), n)] + tracker = defaultdict(int) + comparison = [] + for ngram in ngrams: + while True: + start = sequence.find(ngram, tracker[ngram]) + assert start != -1 # testing the test + + end = start + len(ngram) - 1 + tracker[ngram] = end + 1 + + # ignore partial word matches + if not ( + (start != 0 and sequence[start - 1] != " ") + or (end != len(sequence) - 1 and sequence[end + 1] != " ") + ): + break + + comparison.extend([(ngram, (start, end))]) + + result_to_test = list(word_ngrams_indices(sequence, n)) + assert len(result_to_test) == len(comparison) + assert result_to_test == comparison + + +# Assumptions from GPT3 Paper: +# the 200 characters to remove include punctuation and is actually a half-window + + +# All tests below initially test without any registered contaminants, expecting the same sequence back. +def test_janitor1(): + # First test using a 1gram and expected the first block before the filth to have some remaining + # characters, but the second block should be completely removed. + + sequence = ( + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "FILTH. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + ) + + filth = "filth" + + expected_result = ( + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing " + ) + + janitor = Janitor( + ngram_n=1, window_to_remove=200, too_dirty_cutoff=10, minimum_slice_length=200 + ) + result = janitor.clean_python(sequence) + result = "".join(result) + assert result == sequence + + janitor.register_contaminant(filth) + assert janitor.dirt_ngrams == {filth} + + result = janitor.clean_python(sequence) + result = "".join(result) + assert result == expected_result + + +def test_janitor2(): + # Second test using a 1gram and expected the first block before the filth to have some remaining + # characters, and the second block is longer then 200 characters so should also have some remaining. + + sequence = ( + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "FILTH. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + ) + + filth = "filth" + + janitor = Janitor( + ngram_n=1, window_to_remove=200, too_dirty_cutoff=10, minimum_slice_length=200 + ) + result = janitor.clean_python(sequence) + result = "".join(result) + assert result == sequence + + janitor.register_contaminant(filth) + assert janitor.dirt_ngrams == {filth} + + result = janitor.clean_python(sequence) + result = "".join(result) + assert result == JANITOR_EXPECTED + + +def test_janitor3(): + # Same test as above but with a 6gram. + + sequence = ( + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "FILTH. lots of dirty filtHy FIlTh " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + ) + + janitor = Janitor( + ngram_n=6, window_to_remove=200, too_dirty_cutoff=10, minimum_slice_length=200 + ) + result = janitor.clean_python(sequence) + result = "".join(result) + assert result == sequence + + janitor.register_contaminant(JANITOR_FILTH1) + assert janitor.dirt_ngrams == {JANITOR_FILTH1} + + result = janitor.clean_python(sequence) + result = "".join(result) + assert result == JANITOR_EXPECTED + + +def test_janitor4(): + # This test adds another block to that from the previous. The middle block should be entirely + # removed as the 200 characters are removed from each side. + + sequence = ( + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "FILTH. lots of dirty filtHy FIlTh " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "FILTH. lots of dirty filtHy FIlTh " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + ) + + janitor = Janitor( + ngram_n=6, window_to_remove=200, too_dirty_cutoff=10, minimum_slice_length=200 + ) + result = janitor.clean_python(sequence) + result = "".join(result) + assert result == sequence + + janitor.register_contaminant(JANITOR_FILTH1) + assert janitor.dirt_ngrams == {JANITOR_FILTH1} + + result = janitor.clean_python(sequence) + result = "".join(result) + assert result == JANITOR_EXPECTED + + +def test_janitor5(): + # Same as above but using multiple different filth 6grams. + + sequence = ( + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "FILTH. lots of dirty filtHy FIlTh " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "FILTH. lots of filtHy dirty FIlTh " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + ) + + filths = [JANITOR_FILTH1, JANITOR_FILTH2] + + janitor = Janitor( + ngram_n=6, window_to_remove=200, too_dirty_cutoff=10, minimum_slice_length=200 + ) + result = janitor.clean_python(sequence) + result = "".join(result) + assert result == sequence + + for filth in filths: + janitor.register_contaminant(filth) + assert janitor.dirt_ngrams == set(filths) + + result = janitor.clean_python(sequence) + result = "".join(result) + assert result == JANITOR_EXPECTED + + +def test_janitor6(): + # Same as above but now we add 10 filths and expect the same result, the following test does 11. + + sequence = ( + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "FILTH. lots of dirty filtHy FIlTh " + "FILTH. lots of dirty filtHy FIlTh " + "FILTH. lots of dirty filtHy FIlTh " + "FILTH. lots of dirty filtHy FIlTh " + "FILTH. lots of dirty filtHy FIlTh " + "FILTH. lots of dirty filtHy FIlTh " + "FILTH. lots of dirty filtHy FIlTh " + "FILTH. lots of dirty filtHy FIlTh " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "FILTH. lots of filtHy dirty FIlTh " + "FILTH. lots of filtHy dirty FIlTh " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + ) + + filths = [JANITOR_FILTH1, JANITOR_FILTH2] + + janitor = Janitor( + ngram_n=6, window_to_remove=200, too_dirty_cutoff=10, minimum_slice_length=200 + ) + result = janitor.clean_python(sequence) + result = "".join(result) + assert result == sequence + + for filth in filths: + janitor.register_contaminant(filth) + assert janitor.dirt_ngrams == set(filths) + + result = janitor.clean_python(sequence) + result = "".join(result) + assert result == JANITOR_EXPECTED + + +def test_janitor7(): + # Same as above but now we add 9 filths and expect the same result, the following test does 10. + + sequence = ( + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "FILTH. lots of dirty filtHy FIlTh " + "FILTH. lots of dirty filtHy FIlTh " + "FILTH. lots of dirty filtHy FIlTh " + "FILTH. lots of dirty filtHy FIlTh " + "FILTH. lots of dirty filtHy FIlTh " + "FILTH. lots of dirty filtHy FIlTh " + "FILTH. lots of dirty filtHy FIlTh " + "FILTH. lots of dirty filtHy FIlTh " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "FILTH. lots of filtHy dirty FIlTh " + "FILTH. lots of filtHy dirty FIlTh " + "FILTH. lots of filtHy dirty FIlTh " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + "This is a @line #containing a certain number of characters, 76 to be exact. " + ) + + filths = [JANITOR_FILTH1, JANITOR_FILTH2] + + expected_result = "" + + janitor = Janitor( + ngram_n=6, window_to_remove=200, too_dirty_cutoff=10, minimum_slice_length=200 + ) + result = janitor.clean_python(sequence) + result = "".join(result) + assert result == sequence + + for filth in filths: + janitor.register_contaminant(filth) + assert janitor.dirt_ngrams == set(filths) + + result = janitor.clean_python(sequence) + result = "".join(result) + assert result == expected_result + + +def test_janitor8(): + # This will test the save and load contams + pass diff --git a/lm-evaluation-harness/tests/test_metrics.py b/lm-evaluation-harness/tests/test_metrics.py new file mode 100644 index 0000000000000000000000000000000000000000..1976a497ce045d74af0d5a6a04f7306f95bbb266 --- /dev/null +++ b/lm-evaluation-harness/tests/test_metrics.py @@ -0,0 +1,156 @@ +from lm_eval.api.task import ConfigurableTask, TaskConfig + + +class MockConfigurableTask(ConfigurableTask): + """Mock task for testing metrics""" + + def __init__(self): + # Create a minimal config + config = { + "task": "test_acc_mutual_info", + "output_type": "multiple_choice", + "metric_list": [{"metric": "acc"}, {"metric": "acc_mutual_info"}], + "doc_to_choice": ["A", "B", "C"], + "doc_to_target": 1, # Correct answer is index 1 (choice "B") + "target_delimiter": " ", + } + + # Initialize with minimal setup + self._config = TaskConfig(**config) + self.OUTPUT_TYPE = "multiple_choice" + + # Set up required attributes + self.multiple_input = 0 + self.multiple_target = 0 + + # Set up metrics + self._metric_fn_list = {"acc": None, "acc_mutual_info": None} + self._metric_fn_kwargs = {"acc": {}, "acc_mutual_info": {}} + self._aggregation_list = {} + self._higher_is_better = {} + + def doc_to_choice(self, doc): + return ["A", "B", "C"] + + def doc_to_target(self, doc): + return 1 # Choice "B" is correct + + # Required abstract methods (minimal implementations) + def has_training_docs(self): + return False + + def has_validation_docs(self): + return False + + def has_test_docs(self): + return True + + def download(self, **kwargs): + pass + + +def test_acc_mutual_info_slicing(): + """Test that acc_mutual_info correctly slices conditional and unconditional loglikelihoods""" + + task = MockConfigurableTask() + + # Simulate loglikelihood results for 3 choices + # Format: [(loglikelihood, is_greedy), ...] + # First 3 are conditional P(choice|context), next 3 are unconditional P(choice) + + # Combined results as they would come from the model + # Order: conditional_1, conditional_2, conditional_3, unconditional_1, unconditional_2, unconditional_3 + # Conditional: [-2.0, -1.0, -3.0] - Choice B (index 1) has highest prob + # Unconditional: [-2.5, -2.0, -2.5] - Choice B has higher unconditional prob too + results = [ + (-2.0, False), + (-1.0, True), + (-3.0, False), # Conditional + (-2.5, False), + (-2.0, False), + (-2.5, False), + ] # Unconditional + + # Test the process_results method + doc = {} # Mock document + result_dict = task.process_results(doc, results) + + # Verify that both acc and acc_mutual_info are calculated + assert "acc" in result_dict + assert "acc_mutual_info" in result_dict + + # Both should be 1.0 since choice B (index 1) is correct and has highest probability + assert result_dict["acc"] == 1.0, f"Expected acc=1.0, got {result_dict['acc']}" + assert result_dict["acc_mutual_info"] == 1.0, ( + f"Expected acc_mutual_info=1.0, got {result_dict['acc_mutual_info']}" + ) + + +def test_acc_mutual_info_different_predictions(): + """Test case where conditional and mutual info predictions differ""" + + task = MockConfigurableTask() + + # Mutual info calculation: + # Conditional: A=-1.0, B=-2.0, C=-3.0 (A wins conditionally) + # Unconditional: A=-0.5, B=-2.0, C=-3.0 (A has much higher unconditional prob) + # Mutual info = conditional - unconditional: + # A: -1.0 - (-0.5) = -0.5 + # B: -2.0 - (-2.0) = 0.0 <- B wins with mutual info! + # C: -3.0 - (-3.0) = 0.0 + + results = [ + (-1.0, True), + (-2.0, False), + (-3.0, False), # Conditional (A wins) + (-0.5, False), + (-2.0, False), + (-3.0, False), + ] # Unconditional + + doc = {} + result_dict = task.process_results(doc, results) + + # Regular acc should be 0.0 (A predicted, but B is correct) + assert result_dict["acc"] == 0.0, f"Expected acc=0.0, got {result_dict['acc']}" + + # Mutual info should be 1.0 (B predicted with mutual info, and B is correct) + assert result_dict["acc_mutual_info"] == 1.0, ( + f"Expected acc_mutual_info=1.0, got {result_dict['acc_mutual_info']}" + ) + + +def test_acc_mutual_info_without_metric(): + """Test that normal behavior works when acc_mutual_info is not in metric list""" + + # Create task without acc_mutual_info + config = { + "task": "test_normal", + "output_type": "multiple_choice", + "metric_list": [{"metric": "acc"}], # Only acc, no acc_mutual_info + "doc_to_choice": ["A", "B", "C"], + "doc_to_target": 1, + "target_delimiter": " ", + } + + task = MockConfigurableTask() + task._config = TaskConfig(**config) + task._metric_fn_list = {"acc": None} # Only acc + + # Only conditional loglikelihoods (no unconditional since acc_mutual_info not requested) + results = [(-2.0, False), (-1.0, True), (-3.0, False)] # 3 choices, B wins + + doc = {} + result_dict = task.process_results(doc, results) + + # Should only have acc, not acc_mutual_info + assert "acc" in result_dict + assert "acc_mutual_info" not in result_dict + assert result_dict["acc"] == 1.0 + + +if __name__ == "__main__": + test_acc_mutual_info_slicing() + test_acc_mutual_info_different_predictions() + test_acc_mutual_info_without_metric() + print("All tests passed!") diff --git a/lm-evaluation-harness/tests/test_misc.py b/lm-evaluation-harness/tests/test_misc.py new file mode 100644 index 0000000000000000000000000000000000000000..30267f63d0a518a41ae5681088878edcacc1c729 --- /dev/null +++ b/lm-evaluation-harness/tests/test_misc.py @@ -0,0 +1,14 @@ +import random + +import pytest + +import lm_eval.api.metrics as metrics + + +def test_bootstrapping(): + random.seed(42) + arr = [random.random() for _ in range(1000)] + expected = metrics.mean_stderr(arr) + bootstrapped = metrics.bootstrap_stderr(metrics.mean, arr, iters=100000) + + assert bootstrapped == pytest.approx(expected, abs=1e-4) diff --git a/lm-evaluation-harness/tests/test_prompt.py b/lm-evaluation-harness/tests/test_prompt.py new file mode 100644 index 0000000000000000000000000000000000000000..d1e5deb06507aaf85c1d17d88d6237ff0f042875 --- /dev/null +++ b/lm-evaluation-harness/tests/test_prompt.py @@ -0,0 +1,119 @@ +import random +from typing import List + +import numpy as np +import pytest + +from lm_eval import tasks +from lm_eval.tasks import TaskManager +from lm_eval.utils import join_iters + + +MMLU_ANATOMY_ZERO_SHOT = """The following are multiple choice questions (with answers) about anatomy. + +A lesion causing compression of the facial nerve at the stylomastoid foramen will cause ipsilateral +A. paralysis of the facial muscles. +B. paralysis of the facial muscles and loss of taste. +C. paralysis of the facial muscles, loss of taste and lacrimation. +D. paralysis of the facial muscles, loss of taste, lacrimation and decreased salivation. +Answer:""" + +MMLU_ANATOMY_FIVE_SHOT = """The following are multiple choice questions (with answers) about anatomy. + +What is the embryological origin of the hyoid bone? +A. The first pharyngeal arch +B. The first and second pharyngeal arches +C. The second pharyngeal arch +D. The second and third pharyngeal arches +Answer: D + +Which of these branches of the trigeminal nerve contain somatic motor processes? +A. The supraorbital nerve +B. The infraorbital nerve +C. The mental nerve +D. None of the above +Answer: D + +The pleura +A. have no sensory innervation. +B. are separated by a 2 mm space. +C. extend into the neck. +D. are composed of respiratory epithelium. +Answer: C + +In Angle's Class II Div 2 occlusion there is +A. excess overbite of the upper lateral incisors. +B. negative overjet of the upper central incisors. +C. excess overjet of the upper lateral incisors. +D. excess overjet of the upper central incisors. +Answer: C + +Which of the following is the body cavity that contains the pituitary gland? +A. Abdominal +B. Cranial +C. Pleural +D. Spinal +Answer: B + +A lesion causing compression of the facial nerve at the stylomastoid foramen will cause ipsilateral +A. paralysis of the facial muscles. +B. paralysis of the facial muscles and loss of taste. +C. paralysis of the facial muscles, loss of taste and lacrimation. +D. paralysis of the facial muscles, loss of taste, lacrimation and decreased salivation. +Answer:""" + + +@pytest.mark.parametrize( + "task_names,sets,num_fewshot,seed,num_examples,expected_prompt", + [ + (["mmlu_anatomy"], "test", 0, 42, 1, MMLU_ANATOMY_ZERO_SHOT), + (["mmlu_anatomy"], "test", 5, 42, 1, MMLU_ANATOMY_FIVE_SHOT), + ], +) +def test_mmlu_prompt_rendering( + task_names: List[str], + sets: str, + num_fewshot: int, + seed: int, + num_examples: int, + expected_prompt: str, +): + np.random.seed(seed) + + task_manager = TaskManager() + task_dict = tasks.get_task_dict(task_names, task_manager) + + for task_name, task in task_dict.items(): + if isinstance(task, tuple): + _, task = task + + rnd = random.Random() + rnd.seed(seed) + + iters = [] + + for set in sets.split(","): + docs = None + if set == "train" and task.has_training_docs(): + docs = task.training_docs() + if set == "val" and task.has_validation_docs(): + docs = task.validation_docs() + if set == "test" and task.has_test_docs(): + docs = task.test_docs() + if docs is not None: + iters.append(docs) + + if len(iters) == 0: + raise ValueError + + docs = join_iters(iters) + + for i, doc in ( + zip(range(num_examples), docs) if num_examples > 0 else enumerate(docs) + ): + ctx = task.fewshot_context( + doc=doc, + num_fewshot=num_fewshot, + ) + + assert ctx == expected_prompt diff --git a/lm-evaluation-harness/tests/test_requests_caching.py b/lm-evaluation-harness/tests/test_requests_caching.py new file mode 100644 index 0000000000000000000000000000000000000000..fc520454b92f16e8333ada7a5ad58153199c5e78 --- /dev/null +++ b/lm-evaluation-harness/tests/test_requests_caching.py @@ -0,0 +1,122 @@ +import importlib +import os +import sys +from datetime import datetime +from typing import List, Optional, Tuple + +import pytest +import torch + +from lm_eval.caching.cache import PATH + + +MODULE_DIR = os.path.dirname(os.path.realpath(__file__)) + +# NOTE the script this loads uses simple evaluate +# TODO potentially test both the helper script and the normal script +sys.path.append(f"{MODULE_DIR}/../scripts") +model_loader = importlib.import_module("requests_caching") +run_model_for_task_caching = model_loader.run_model_for_task_caching + +os.environ["HF_DATASETS_TRUST_REMOTE_CODE"] = "1" +DEFAULT_TASKS = ["lambada_openai", "sciq"] + + +@pytest.fixture(autouse=True) +def setup_and_teardown(): + # Setup + torch.use_deterministic_algorithms(False) + clear_cache() + # Yields control back to the test function + yield + # Cleanup here + + +def clear_cache(): + if os.path.exists(PATH): + cache_files = os.listdir(PATH) + for file in cache_files: + file_path = f"{PATH}/{file}" + os.unlink(file_path) + + +# leaving tasks here to allow for the option to select specific task files +def get_cache_files(tasks: Optional[List[str]] = None) -> Tuple[List[str], List[str]]: + cache_files = os.listdir(PATH) + + file_task_names = [] + + for file in cache_files: + file_without_prefix = file.split("-")[1] + file_without_prefix_and_suffix = file_without_prefix.split(".")[0] + file_task_names.extend([file_without_prefix_and_suffix]) + + return cache_files, file_task_names + + +def assert_created(tasks: List[str], file_task_names: List[str]): + tasks.sort() + file_task_names.sort() + + assert tasks == file_task_names + + +@pytest.mark.parametrize("tasks", [DEFAULT_TASKS]) +def requests_caching_true(tasks: List[str]): + run_model_for_task_caching(tasks=tasks, cache_requests="true") + + cache_files, file_task_names = get_cache_files() + print(file_task_names) + assert_created(tasks=tasks, file_task_names=file_task_names) + + +@pytest.mark.parametrize("tasks", [DEFAULT_TASKS]) +def requests_caching_refresh(tasks: List[str]): + run_model_for_task_caching(tasks=tasks, cache_requests="true") + + timestamp_before_test = datetime.now().timestamp() + + run_model_for_task_caching(tasks=tasks, cache_requests="refresh") + + cache_files, file_task_names = get_cache_files() + + for file in cache_files: + modification_time = os.path.getmtime(f"{PATH}/{file}") + assert modification_time > timestamp_before_test + + tasks.sort() + file_task_names.sort() + + assert tasks == file_task_names + + +@pytest.mark.parametrize("tasks", [DEFAULT_TASKS]) +def requests_caching_delete(tasks: List[str]): + # populate the data first, rerun this test within this test for additional confidence + # test_requests_caching_true(tasks=tasks) + + run_model_for_task_caching(tasks=tasks, cache_requests="delete") + + cache_files, file_task_names = get_cache_files() + + assert len(cache_files) == 0 + + +# useful for locally running tests through the debugger +if __name__ == "__main__": + + def run_tests(): + tests = [ + # test_requests_caching_true, + # test_requests_caching_refresh, + # test_requests_caching_delete, + ] + # Lookups of global names within a loop is inefficient, so copy to a local variable outside of the loop first + default_tasks = DEFAULT_TASKS + for test_func in tests: + clear_cache() + test_func(tasks=default_tasks) + + print("Tests pass") + + run_tests() diff --git a/lm-evaluation-harness/tests/test_task_manager.py b/lm-evaluation-harness/tests/test_task_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..00cecc8fd3ca28ed940ca541a8ffc6f9b1d6f448 --- /dev/null +++ b/lm-evaluation-harness/tests/test_task_manager.py @@ -0,0 +1,73 @@ +import tempfile +from pathlib import Path + +import pytest + +from lm_eval.tasks import TaskManager + + +@pytest.fixture(scope="module") +def custom_task_name(): + return "zzz_my_python_task" + + +@pytest.fixture(scope="module") +def custom_task_tag(): + return "zzz-tag" + + +@pytest.fixture(scope="module") +def task_yaml(pytestconfig, custom_task_name, custom_task_tag): + yield f"""include: {pytestconfig.rootpath}/lm_eval/tasks/arc/arc_easy.yaml +task: {custom_task_name} +class: !function {custom_task_name}.MockPythonTask +tag: + - {custom_task_tag} +""" + + +@pytest.fixture(scope="module") +def task_code(): + return """ +from lm_eval.tasks import ConfigurableTask + +class MockPythonTask(ConfigurableTask): + + def __init__( + self, + data_dir=None, + cache_dir=None, + download_mode=None, + config=None, + ) -> None: + config.pop("class") + super().__init__(data_dir, cache_dir, download_mode, config) +""" + + +@pytest.fixture(scope="module") +def custom_task_files_dir(task_yaml, task_code, custom_task_name): + with tempfile.TemporaryDirectory() as temp_dir: + yaml_path = Path(temp_dir) / f"{custom_task_name}.yaml" + with open(yaml_path, "w") as f: + f.write(task_yaml) + pysource_path = Path(temp_dir) / f"{custom_task_name}.py" + with open(pysource_path, "w") as f: + f.write(task_code) + yield temp_dir + + +def test_python_task_inclusion( + custom_task_files_dir: Path, custom_task_name: str, custom_task_tag: str +): + task_manager = TaskManager( + verbosity="INFO", include_path=str(custom_task_files_dir) + ) + # check if python tasks enters the global task_index + assert custom_task_name in task_manager.task_index + # check if subtask is present + assert custom_task_name in task_manager.all_subtasks + # check if tag is present + assert custom_task_tag in task_manager.all_tags + # check if it can be loaded by tag (custom_task_tag) + assert custom_task_name in task_manager.load_task_or_group(custom_task_tag) diff --git a/lm-evaluation-harness/tests/test_tasks.py b/lm-evaluation-harness/tests/test_tasks.py new file mode 100644 index 0000000000000000000000000000000000000000..df67c88c9cf88e8436a223f510db6cd89dae0160 --- /dev/null +++ b/lm-evaluation-harness/tests/test_tasks.py @@ -0,0 +1,210 @@ +import os +from itertools import islice + +import datasets +import pytest + +import lm_eval.tasks as tasks +from lm_eval.api.task import ConfigurableTask +from lm_eval.evaluator_utils import get_task_list + +from .utils import new_tasks + + +datasets.config.HF_DATASETS_TRUST_REMOTE_CODE = True +os.environ["TOKENIZERS_PARALLELISM"] = "false" +# Default Task +TASKS = ["arc_easy"] + + +def get_new_tasks_else_default(): + """ + Check if any modifications have been made to built-in tasks and return + the list, otherwise return the default task list + """ + global TASKS + # CI: new_tasks checks if any modifications have been made + task_classes = new_tasks() + # Check if task_classes is empty + return task_classes if task_classes else TASKS + + +def task_class(task_names=None, task_manager=None) -> ConfigurableTask: + """ + Convert a list of task names to a list of ConfigurableTask instances + """ + if task_manager is None: + task_manager = tasks.TaskManager() + res = tasks.get_task_dict(task_names, task_manager) + res = [x.task for x in get_task_list(res)] + + return res + + +@pytest.fixture() +def limit() -> int: + return 10 + + +# Tests +class BaseTasks: + """ + Base class for testing tasks + """ + + def test_download(self, task_class: ConfigurableTask): + task_class.download() + assert task_class.dataset is not None + + def test_has_training_docs(self, task_class: ConfigurableTask): + assert task_class.has_training_docs() in [True, False] + + def test_check_training_docs(self, task_class: ConfigurableTask): + if task_class.has_training_docs(): + assert task_class._config["training_split"] is not None + + def test_has_validation_docs(self, task_class): + assert task_class.has_validation_docs() in [True, False] + + def test_check_validation_docs(self, task_class): + if task_class.has_validation_docs(): + assert task_class._config["validation_split"] is not None + + def test_has_test_docs(self, task_class): + assert task_class.has_test_docs() in [True, False] + + def test_check_test_docs(self, task_class): + task = task_class + if task.has_test_docs(): + assert task._config["test_split"] is not None + + def test_should_decontaminate(self, task_class): + task = task_class + assert task.should_decontaminate() in [True, False] + if task.should_decontaminate(): + assert task._config["doc_to_decontamination_query"] is not None + + def test_doc_to_text(self, task_class, limit): + task = task_class + arr = ( + list(islice(task.test_docs(), limit)) + if task.has_test_docs() + else list(islice(task.validation_docs(), limit)) + ) + _array = [task.doc_to_text(doc) for doc in arr] + # space convention; allow txt to have length 0 for perplexity-like tasks since the model tacks an <|endoftext|> on + target_delimiter: str = task.config.target_delimiter + if not task.multiple_input: + for x in _array: + assert isinstance(x, str) + assert ( + (x[-1].isspace() is False if len(x) > 0 else True) + if target_delimiter.isspace() + else True + ), ( + "doc_to_text ends in a whitespace and target delimiter also a whitespace" + ) + else: + pass + + def test_create_choices(self, task_class, limit): + task = task_class + arr = ( + list(islice(task.test_docs(), limit)) + if task.has_test_docs() + else list(islice(task.validation_docs(), limit)) + ) + if "multiple_choice" in task._config.output_type: + _array = [task.doc_to_choice(doc) for doc in arr] + assert all(isinstance(x, list) for x in _array) + assert all(isinstance(x[0], str) for x in _array) + + def test_doc_to_target(self, task_class, limit): + task = task_class + arr = ( + list(islice(task.test_docs(), limit)) + if task.has_test_docs() + else list(islice(task.validation_docs(), limit)) + ) + _array_target = [task.doc_to_target(doc) for doc in arr] + if task._config.output_type == "multiple_choice": + # TODO: label can be string or int; add better test conditions + assert all( + (isinstance(label, int) or isinstance(label, str)) + for label in _array_target + ) + + def test_build_all_requests(self, task_class, limit): + task_class.build_all_requests(rank=1, limit=limit, world_size=1) + assert task_class.instances is not None + + # ToDO: Add proper testing + def test_construct_requests(self, task_class, limit): + task = task_class + arr = ( + list(islice(task.test_docs(), limit)) + if task.has_test_docs() + else list(islice(task.validation_docs(), limit)) + ) + # ctx is "" for multiple input tasks + requests = [ + task.construct_requests( + doc=doc, ctx="" if task.multiple_input else task.doc_to_text(doc) + ) + for doc in arr + ] + assert len(requests) == limit if limit else True + + +@pytest.mark.parametrize( + "task_class", + task_class(get_new_tasks_else_default()), + ids=lambda x: f"{x.config.task}", +) +class TestNewTasksElseDefault(BaseTasks): + """ + Test class parameterized with a list of new/modified tasks + (or a set of default tasks if none have been modified) + """ + + +@pytest.mark.parametrize( + "task_class", + task_class( + ["arc_easy_unitxt"], tasks.TaskManager(include_path="./tests/testconfigs") + ), + ids=lambda x: f"{x.config.task}", +) +class TestUnitxtTasks(BaseTasks): + """ + Test class for Unitxt tasks parameterized with a small custom + task as described here: + https://www.unitxt.ai/en/latest/docs/lm_eval.html + """ + + def test_check_training_docs(self, task_class: ConfigurableTask): + if task_class.has_training_docs(): + assert task_class.dataset["train"] is not None + + def test_check_validation_docs(self, task_class): + if task_class.has_validation_docs(): + assert task_class.dataset["validation"] is not None + + def test_check_test_docs(self, task_class): + task = task_class + if task.has_test_docs(): + assert task.dataset["test"] is not None + + def test_doc_to_text(self, task_class, limit: int): + task = task_class + arr = ( + list(islice(task.test_docs(), limit)) + if task.has_test_docs() + else list(islice(task.validation_docs(), limit)) + ) + _array = [task.doc_to_text(doc) for doc in arr] + if not task.multiple_input: + for x in _array: + assert isinstance(x, str) + else: + pass diff --git a/lm-evaluation-harness/tests/test_utils.py b/lm-evaluation-harness/tests/test_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..dfa8741aefdc7a49eda07ae483e411be3e3ec2e1 --- /dev/null +++ b/lm-evaluation-harness/tests/test_utils.py @@ -0,0 +1,398 @@ +import itertools + +import numpy as np +import pytest +import torch + +from lm_eval.api.metrics import ( + aggregate_subtask_metrics, + mean, + pooled_sample_stderr, + stderr_for_metric, +) +from lm_eval.models.utils import Collator +from lm_eval.utils import ( + get_rolling_token_windows, + make_disjoint_window, +) + + +# noinspection DuplicatedCode +def test_get_rolling_token_windows_v1(): + gold = [ + ([-100, 0, 1, 2, 3, 4, 5, 6, 7, 8], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]), + ( + [9, 10, 11, 12, 13, 14, 15, 16, 17, 18], + [10, 11, 12, 13, 14, 15, 16, 17, 18, 19], + ), + ( + [19, 20, 21, 22, 23, 24, 25, 26, 27, 28], + [20, 21, 22, 23, 24, 25, 26, 27, 28, 29], + ), + ([23, 24, 25, 26, 27, 28, 29, 30, 31, 32], [30, 31, 32, 33]), + ] + x = list(range(34)) + generator = get_rolling_token_windows( + token_list=x, + prefix_token=-100, + max_seq_len=10, + context_len=1, + ) + pred_length = 0 + output = [] + for input_tokens, pred_tokens in generator: + output.extend([(input_tokens, pred_tokens)]) + pred_length += len(pred_tokens) + assert pred_length == len(x) + assert gold == output + + +# noinspection DuplicatedCode +def test_get_rolling_token_windows_v2(): + gold = [ + ([-100, 0, 1, 2, 3, 4, 5, 6, 7, 8], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]), + ([2, 3, 4, 5, 6, 7, 8, 9, 10, 11], [10, 11, 12]), + ([5, 6, 7, 8, 9, 10, 11, 12, 13, 14], [13, 14, 15]), + ([8, 9, 10, 11, 12, 13, 14, 15, 16, 17], [16, 17, 18]), + ([11, 12, 13, 14, 15, 16, 17, 18, 19, 20], [19, 20, 21]), + ([14, 15, 16, 17, 18, 19, 20, 21, 22, 23], [22, 23, 24]), + ([17, 18, 19, 20, 21, 22, 23, 24, 25, 26], [25, 26, 27]), + ([20, 21, 22, 23, 24, 25, 26, 27, 28, 29], [28, 29, 30]), + ([23, 24, 25, 26, 27, 28, 29, 30, 31, 32], [31, 32, 33]), + ] + x = list(range(34)) + generator = get_rolling_token_windows( + token_list=x, + prefix_token=-100, + max_seq_len=10, + context_len=8, + ) + pred_length = 0 + output = [] + for input_tokens, pred_tokens in generator: + output.extend([(input_tokens, pred_tokens)]) + pred_length += len(pred_tokens) + assert pred_length == len(x) + assert gold == output + + +# noinspection DuplicatedCode +def test_get_rolling_token_windows_v3(): + gold = [ + ([-100, 0, 1, 2, 3, 4, 5, 6, 7, 8], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]), + ([0, 1, 2, 3, 4, 5, 6, 7, 8, 9], [10]), + ([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [11]), + ([2, 3, 4, 5, 6, 7, 8, 9, 10, 11], [12]), + ([3, 4, 5, 6, 7, 8, 9, 10, 11, 12], [13]), + ([4, 5, 6, 7, 8, 9, 10, 11, 12, 13], [14]), + ([5, 6, 7, 8, 9, 10, 11, 12, 13, 14], [15]), + ([6, 7, 8, 9, 10, 11, 12, 13, 14, 15], [16]), + ([7, 8, 9, 10, 11, 12, 13, 14, 15, 16], [17]), + ([8, 9, 10, 11, 12, 13, 14, 15, 16, 17], [18]), + ([9, 10, 11, 12, 13, 14, 15, 16, 17, 18], [19]), + ([10, 11, 12, 13, 14, 15, 16, 17, 18, 19], [20]), + ([11, 12, 13, 14, 15, 16, 17, 18, 19, 20], [21]), + ([12, 13, 14, 15, 16, 17, 18, 19, 20, 21], [22]), + ([13, 14, 15, 16, 17, 18, 19, 20, 21, 22], [23]), + ([14, 15, 16, 17, 18, 19, 20, 21, 22, 23], [24]), + ([15, 16, 17, 18, 19, 20, 21, 22, 23, 24], [25]), + ([16, 17, 18, 19, 20, 21, 22, 23, 24, 25], [26]), + ([17, 18, 19, 20, 21, 22, 23, 24, 25, 26], [27]), + ([18, 19, 20, 21, 22, 23, 24, 25, 26, 27], [28]), + ([19, 20, 21, 22, 23, 24, 25, 26, 27, 28], [29]), + ([20, 21, 22, 23, 24, 25, 26, 27, 28, 29], [30]), + ([21, 22, 23, 24, 25, 26, 27, 28, 29, 30], [31]), + ([22, 23, 24, 25, 26, 27, 28, 29, 30, 31], [32]), + ([23, 24, 25, 26, 27, 28, 29, 30, 31, 32], [33]), + ] + x = list(range(34)) + generator = get_rolling_token_windows( + token_list=x, + prefix_token=-100, + max_seq_len=10, + context_len=10, + ) + pred_length = 0 + output = [] + for input_tokens, pred_tokens in generator: + output.extend([(input_tokens, pred_tokens)]) + pred_length += len(pred_tokens) + assert pred_length == len(x) + assert gold == output + + +# noinspection DuplicatedCode +def test_get_rolling_token_windows_v4(): + gold = [ + ([-100, 0, 1, 2, 3, 4, 5, 6, 7, 8], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]), + ([0, 1, 2, 3, 4, 5, 6, 7, 8, 9], [10]), + ([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [11]), + ([2, 3, 4, 5, 6, 7, 8, 9, 10, 11], [12]), + ([3, 4, 5, 6, 7, 8, 9, 10, 11, 12], [13]), + ([4, 5, 6, 7, 8, 9, 10, 11, 12, 13], [14]), + ([5, 6, 7, 8, 9, 10, 11, 12, 13, 14], [15]), + ([6, 7, 8, 9, 10, 11, 12, 13, 14, 15], [16]), + ([7, 8, 9, 10, 11, 12, 13, 14, 15, 16], [17]), + ([8, 9, 10, 11, 12, 13, 14, 15, 16, 17], [18]), + ([9, 10, 11, 12, 13, 14, 15, 16, 17, 18], [19]), + ([10, 11, 12, 13, 14, 15, 16, 17, 18, 19], [20]), + ([11, 12, 13, 14, 15, 16, 17, 18, 19, 20], [21]), + ([12, 13, 14, 15, 16, 17, 18, 19, 20, 21], [22]), + ([13, 14, 15, 16, 17, 18, 19, 20, 21, 22], [23]), + ([14, 15, 16, 17, 18, 19, 20, 21, 22, 23], [24]), + ([15, 16, 17, 18, 19, 20, 21, 22, 23, 24], [25]), + ([16, 17, 18, 19, 20, 21, 22, 23, 24, 25], [26]), + ([17, 18, 19, 20, 21, 22, 23, 24, 25, 26], [27]), + ([18, 19, 20, 21, 22, 23, 24, 25, 26, 27], [28]), + ([19, 20, 21, 22, 23, 24, 25, 26, 27, 28], [29]), + ] + x = list(range(30)) + generator = get_rolling_token_windows( + token_list=x, + prefix_token=-100, + max_seq_len=10, + context_len=10, + ) + pred_length = 0 + output = [] + for input_tokens, pred_tokens in generator: + output.extend([(input_tokens, pred_tokens)]) + pred_length += len(pred_tokens) + assert pred_length == len(x) + assert gold == output + + +# noinspection DuplicatedCode +def test_get_rolling_token_windows_v5(): + gold = [ + ([-100, 0, 1, 2, 3, 4, 5, 6, 7, 8], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]), + ( + [9, 10, 11, 12, 13, 14, 15, 16, 17, 18], + [10, 11, 12, 13, 14, 15, 16, 17, 18, 19], + ), + ( + [19, 20, 21, 22, 23, 24, 25, 26, 27, 28], + [20, 21, 22, 23, 24, 25, 26, 27, 28, 29], + ), + ] + x = list(range(30)) + generator = get_rolling_token_windows( + token_list=x, + prefix_token=-100, + max_seq_len=10, + context_len=1, + ) + pred_length = 0 + output = [] + for input_tokens, pred_tokens in generator: + output.extend([(input_tokens, pred_tokens)]) + pred_length += len(pred_tokens) + assert pred_length == len(x) + assert gold == output + + +# noinspection DuplicatedCode +def test_get_rolling_token_windows_v6(): + gold = [ + ([-100, 0], [0, 1]), + ([1, 2], [2, 3]), + ([3, 4], [4, 5]), + ([5, 6], [6, 7]), + ([6, 7], [8]), + ] + x = list(range(9)) + generator = get_rolling_token_windows( + token_list=x, + prefix_token=-100, + max_seq_len=2, + context_len=1, + ) + pred_length = 0 + output = [] + for input_tokens, pred_tokens in generator: + output.extend([(input_tokens, pred_tokens)]) + pred_length += len(pred_tokens) + assert pred_length == len(x) + assert gold == output + + +def test_get_rolling_token_windows_empty(): + generator = get_rolling_token_windows( + token_list=[], + prefix_token=-100, + max_seq_len=2, + context_len=1, + ) + n = 0 + for _ in generator: + n += 1 + assert n == 0 + + +def test_make_disjoint_window(): + assert make_disjoint_window(([1, 2, 3, 4, 5], [2, 3, 4, 5, 6])) == ( + [1], + [2, 3, 4, 5, 6], + ) + assert make_disjoint_window(([1, 2, 3, 4, 5], [4, 5, 6])) == ([1, 2, 3], [4, 5, 6]) + assert make_disjoint_window(([1, 2, 3, 4, 5], [6])) == ([1, 2, 3, 4, 5], [6]) + + +class TestCollator: + def make_generate_sample(self, end=10): + strings = ["x" * i for i in range(1, end + 1)] + gen_kwargs1, gen_kwargs2 = ( + {"temperature": 0}, + {"temperature": 0, "until": ["nn", "\n\n"]}, + ) + args = [ + (string, gen_kwargs1 if i < len(strings) // 2 else gen_kwargs2) + for i, string in enumerate(strings) + ] + + return args + + def make_loglikelihood_sample(self, end=11): + samples = [ + (("x", "x"), list(range(1, total_length + 1))) + for total_length in range(1, end + 1) + ] + return samples + + def make_loglikelihood_sample_group(self, end=11): + a = [(("x", "x"), [1, 2, 3, 4, 5, 6, 7, 8], [x]) for x in range(9)] + b = [ + (("x", "x"), [1, 2, 3, 4, 5, 6, 7, 8], [x, y, z]) + for x, y, z in zip(range(9), range(9, 18), range(18, 27)) + ] + return a + b + + @pytest.mark.parametrize("batch_size, end", [(17, 30), (8, 61), (12, 48), (0, 9)]) + def test_generations(self, batch_size, end): + _collate_gen = lambda x: (-len(x[0]), x[0]) # noqa: E731 + + generation_samples = self.make_generate_sample(int(end)) + gens = Collator(generation_samples, _collate_gen, group_by="gen_kwargs") + chunks_gen = gens.get_batched(n=int(batch_size), batch_fn=None) + output = [] + group_one = end // 2 + group_two = end - end // 2 + is_batch = batch_size != 0 + for chunks in chunks_gen: + # check batching + assert ( + len(chunks) <= batch_size + if is_batch + else len(chunks) in [group_one, group_two] + ) + # check if reorder-er is working correctly + chunk_lengths = [len(chunk[0]) for chunk in chunks] + assert chunk_lengths == sorted(chunk_lengths, reverse=True) + # check if grouping correctly + chunk_to_compare = chunks[0][1] + assert all(x[1] == chunk_to_compare for x in chunks) + for x in chunks: + output.extend([x]) + reordered_output = gens.get_original(output) + # check get original + assert reordered_output == generation_samples + + @pytest.mark.parametrize("batch_size, end", [(17, 30), (8, 61), (12, 48), (0, 3)]) + def test_loglikelihood(self, batch_size, end): + _collate_log = lambda x: (-len(x[1]), tuple(x[1])) # noqa: E731 + loglikelihood_samples = self.make_loglikelihood_sample(int(end)) + loglikelihoods = Collator( + loglikelihood_samples, + _collate_log, + ) + chunks_gen = loglikelihoods.get_batched(n=int(batch_size), batch_fn=None) + output = [] + is_batch = batch_size != 0 + for chunks in chunks_gen: + # check batching + assert len(chunks) <= batch_size if is_batch else len(chunks) == end + # check reorder + chunk_lengths = [len(chunk[1]) for chunk in chunks] + assert chunk_lengths == sorted(chunk_lengths, reverse=True) + for x in chunks: + output.extend([x[1]]) + # check indices + reordered_output = loglikelihoods.get_original(output) + assert reordered_output == [x[1] for x in loglikelihood_samples] + + @pytest.mark.parametrize("batch_size", [17, 8, 12, 0]) + def test_context_grouping(self, batch_size): + def _collate(x): + toks = x[1] + x[2] + return -len(toks), tuple(toks) + + _collate_log = _collate # noqa: E731 + loglikelihood_samples = self.make_loglikelihood_sample_group() + loglikelihoods = Collator( + loglikelihood_samples, + _collate_log, + group_fn=lambda a: a[-2] + a[-1][:-1], + group_by="contexts", + ) + chunks_gen = loglikelihoods.get_batched(n=int(batch_size), batch_fn=None) + output = [] + outputs_ = [] + is_batch = batch_size != 0 + for chunks in chunks_gen: + # check batching + if is_batch: + assert len(chunks) <= batch_size + # check reorder + chunk_lengths = [len(chunk[1]) for chunk in chunks] + assert chunk_lengths == sorted(chunk_lengths, reverse=True) + for x in chunks: + for request_str, cont_toks, logits in loglikelihoods.get_cache( + req_str="".join(x[0]), + cxt_toks=x[1], + cont_toks=x[2], + logits=torch.tensor([1, 2, 3, 4, 5, 6, 7, 8]) + .unsqueeze(0) + .unsqueeze(0), + ): + output.extend([x[1]]) + outputs_.extend([cont_toks]) + assert len(output) == len(outputs_) + # check indices + reordered_output = loglikelihoods.get_original(output) + assert reordered_output == [x[1] for x in loglikelihood_samples] + + +def test_aggregate_mean(): + # test weight_by_size is respected + assert ( + aggregate_subtask_metrics([0.3, 0.2, 0.4], [20, 40, 100], weight_by_size=False) + == 0.3 + ) + assert ( + aggregate_subtask_metrics([0.3, 0.2, 0.4], [20, 40, 100], weight_by_size=True) + == 0.3375 + ) + + +@pytest.mark.parametrize( + "samples", + [ + [40 * [1.0] + 60 * [0.0], 30 * [1.0] + 30 * [0.0], 20 * [1.0] + 60 * [0.0]], + [35 * [1.0] + 65 * [0.0], 20 * [1.0] + 20 * [0.0]], + ], +) +def test_aggregate_stderrs(samples): + # check that aggregating subtasks' bootstrap stderrs with our formula + # (using weight_by_size) is ~equiv. + # to just getting bootstrap stderr of the whole set of samples + mean_stderr = stderr_for_metric(metric=mean, bootstrap_iters=100000) + + stderrs = [mean_stderr(subtask) for subtask in samples] + + sizes = [len(subtask) for subtask in samples] + + assert np.allclose( + pooled_sample_stderr(stderrs, sizes), + mean_stderr(list(itertools.chain.from_iterable(samples))), + atol=1.0e-3, + ) diff --git a/lm-evaluation-harness/tests/testconfigs/arc_easy_unitxt.yaml b/lm-evaluation-harness/tests/testconfigs/arc_easy_unitxt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..65723008643cbc543832c5ce9b8e6fbb82049ce3 --- /dev/null +++ b/lm-evaluation-harness/tests/testconfigs/arc_easy_unitxt.yaml @@ -0,0 +1,3 @@ +task: arc_easy_unitxt +include: ../../lm_eval/tasks/unitxt/unitxt +recipe: card=cards.ai2_arc.arc_easy,template=templates.qa.multiple_choice.open.all diff --git a/lm-evaluation-harness/tests/testconfigs/arc_test.yaml b/lm-evaluation-harness/tests/testconfigs/arc_test.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c9052dc80d6d6c3cd26944feec21cfb673330b41 --- /dev/null +++ b/lm-evaluation-harness/tests/testconfigs/arc_test.yaml @@ -0,0 +1,21 @@ +task: arc_easy +dataset_path: allenai/ai2_arc +dataset_name: ARC-Easy +output_type: multiple_choice +training_split: train +validation_split: validation +test_split: test +doc_to_text: "Question: {{question}}\nAnswer:" +doc_to_target: "{{choices.label.index(answerKey)}}" +doc_to_choice: "{{choices.text}}" +should_decontaminate: true +doc_to_decontamination_query: "Question: {{question}}\nAnswer:" +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm-evaluation-harness/tests/testconfigs/sae_lens_intervention.csv b/lm-evaluation-harness/tests/testconfigs/sae_lens_intervention.csv new file mode 100644 index 0000000000000000000000000000000000000000..2c6aaa40c21ae09b451f3c68de4620ceabd6ab6d --- /dev/null +++ b/lm-evaluation-harness/tests/testconfigs/sae_lens_intervention.csv @@ -0,0 +1,2 @@ +loader,hook_action,sparse_model,hookpoint,feature_index,steering_coefficient,sae_id,description, +sae_lens,add,gemma-scope-2b-pt-res-canonical,layers.20,12082,10.0,layer_20/width_16k/canonical,increase dogs, diff --git a/lm-evaluation-harness/tests/testconfigs/sparsify_intervention.csv b/lm-evaluation-harness/tests/testconfigs/sparsify_intervention.csv new file mode 100644 index 0000000000000000000000000000000000000000..96ccf341d37e647db2d7ecd81080635518ff90f3 --- /dev/null +++ b/lm-evaluation-harness/tests/testconfigs/sparsify_intervention.csv @@ -0,0 +1,2 @@ +loader,hook_action,sparse_model,hookpoint,feature_index,steering_coefficient,sae_id,description, +sparsify,add,EleutherAI/sae-pythia-70m-32k,layers.3,30,0.1,,, diff --git a/lm-evaluation-harness/tests/testdata/ai2_arc_10_hf_pretrained-EleutherAI-pythia-14m-dtype-float32-device-cpu.txt b/lm-evaluation-harness/tests/testdata/ai2_arc_10_hf_pretrained-EleutherAI-pythia-14m-dtype-float32-device-cpu.txt new file mode 100644 index 0000000000000000000000000000000000000000..6ba01f21d5711e90b22bafd9979f7a517b98b0ef --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/ai2_arc_10_hf_pretrained-EleutherAI-pythia-14m-dtype-float32-device-cpu.txt @@ -0,0 +1,6 @@ +| Tasks |Version|Filter|n-shot| Metric | |Value| |Stderr| +|-------------|------:|------|-----:|--------|---|----:|---|------| +|arc_challenge| 1|none | 0|acc |↑ | 0.0|± | N/A| +| | |none | 0|acc_norm|↑ | 0.0|± | N/A| +|arc_easy | 1|none | 0|acc |↑ | 0.3|± | N/A| +| | |none | 0|acc_norm|↑ | 0.1|± | N/A| \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/anagrams1-v0-greedy_until b/lm-evaluation-harness/tests/testdata/anagrams1-v0-greedy_until new file mode 100644 index 0000000000000000000000000000000000000000..55364250028072b1f238b095c4c3eb9373a4a280 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/anagrams1-v0-greedy_until @@ -0,0 +1 @@ +7c0c5246d3f751f39119a5629ac1d4b2c6fd2a315f78d6de9b2c387e24e3fef1 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/anagrams1-v0-res.json b/lm-evaluation-harness/tests/testdata/anagrams1-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..c89528892ae2cb5dfc87cf28f587062a18323d87 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/anagrams1-v0-res.json @@ -0,0 +1 @@ +{"results": {"anagrams1": {"acc": 0.0, "acc_stderr": 0.0}}, "versions": {"anagrams1": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/anagrams2-v0-greedy_until b/lm-evaluation-harness/tests/testdata/anagrams2-v0-greedy_until new file mode 100644 index 0000000000000000000000000000000000000000..9db9d158dc07c46ddb5bc88ea797cc41080ca941 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/anagrams2-v0-greedy_until @@ -0,0 +1 @@ +6700a3c44e48abe8337238dcbe3b54cf4abafe0c204c52d921e590872fbd05e7 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/anagrams2-v0-res.json b/lm-evaluation-harness/tests/testdata/anagrams2-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..f74887fe16ec042fcdf995b7b7b694d3fec92659 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/anagrams2-v0-res.json @@ -0,0 +1 @@ +{"results": {"anagrams2": {"acc": 0.0, "acc_stderr": 0.0}}, "versions": {"anagrams2": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/anli_r1-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/anli_r1-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..4450c0628e9d9a6f8ff90c9efa0c5e5b1b7e4069 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/anli_r1-v0-loglikelihood @@ -0,0 +1 @@ +3a84baf2f170e138c6ce0bc9f06f905def35d705fa2b8781f10c87aef404c4cb \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/anli_r1-v0-res.json b/lm-evaluation-harness/tests/testdata/anli_r1-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..b6f6b350182f2a363b2b247b1145dd5c3b54157e --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/anli_r1-v0-res.json @@ -0,0 +1 @@ +{"results": {"anli_r1": {"acc": 0.334, "acc_stderr": 0.014922019523732967}}, "versions": {"anli_r1": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/anli_r2-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/anli_r2-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..4a437fc8a8bb7928fade05baac9319b74d939bf8 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/anli_r2-v0-loglikelihood @@ -0,0 +1 @@ +d0ea3c3e09d533982c15b4c034439896d6af4bbafb2254d305e20215534a251d \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/anli_r2-v0-res.json b/lm-evaluation-harness/tests/testdata/anli_r2-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..6dc08ebbaa852afef27dbd6002575ada16870eb0 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/anli_r2-v0-res.json @@ -0,0 +1 @@ +{"results": {"anli_r2": {"acc": 0.356, "acc_stderr": 0.015149042659306628}}, "versions": {"anli_r2": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/anli_r3-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/anli_r3-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..29d3d67c8b038c0b0882e97071033fefb9481a41 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/anli_r3-v0-loglikelihood @@ -0,0 +1 @@ +6b6e5c6a794f2fbff78b7aa24fe0c90156039334bbd1cb34f7af9fc6e6183845 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/anli_r3-v0-res.json b/lm-evaluation-harness/tests/testdata/anli_r3-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..548dea1e2285461362f32707937ff84f37572957 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/anli_r3-v0-res.json @@ -0,0 +1 @@ +{"results": {"anli_r3": {"acc": 0.31916666666666665, "acc_stderr": 0.01346230971200514}}, "versions": {"anli_r3": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/arc_challenge-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/arc_challenge-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..91a3560635db37739cd7504bdc84c6c840192462 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/arc_challenge-v0-loglikelihood @@ -0,0 +1 @@ +41c34c96cca8ace661911d0033d630c554b283f5a3953bcdc50720ae6b00a9c1 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/arc_challenge-v0-res.json b/lm-evaluation-harness/tests/testdata/arc_challenge-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..49f34a73061139ee50a27896a5de9d0f1613941c --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/arc_challenge-v0-res.json @@ -0,0 +1 @@ +{"results": {"arc_challenge": {"acc": 0.24488054607508533, "acc_norm": 0.2440273037542662, "acc_norm_stderr": 0.012551447627856257, "acc_stderr": 0.012566273985131354}}, "versions": {"arc_challenge": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/arc_challenge-v2.0-res.json b/lm-evaluation-harness/tests/testdata/arc_challenge-v2.0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..f0186c4c4b395b6f57e26120975ec0378cd9c0ea --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/arc_challenge-v2.0-res.json @@ -0,0 +1 @@ +{"results": {"arc_challenge": {"acc": 0.26621160409556316, "acc_norm": 0.28242320819112626, "acc_norm_stderr": 0.01315545688409722, "acc_stderr": 0.01291577478152323}}, "versions": {"arc_challenge": "2.0"}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/arc_easy-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/arc_easy-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..d82be433abe592079dc9ce67ec7e97fe668c8590 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/arc_easy-v0-loglikelihood @@ -0,0 +1 @@ +ffa6e39a35a16299dcb015f17f986aaa598ad8b4840c4cebe0339a7042232741 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/arc_easy-v0-res.json b/lm-evaluation-harness/tests/testdata/arc_easy-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..f217448594199a54d671be7302857509eb6d691f --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/arc_easy-v0-res.json @@ -0,0 +1 @@ +{"results": {"arc_easy": {"acc": 0.2474747474747475, "acc_norm": 0.24074074074074073, "acc_norm_stderr": 0.008772796145221907, "acc_stderr": 0.008855114414834707}}, "versions": {"arc_easy": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/arithmetic_1dc-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/arithmetic_1dc-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..01756b4d47703cc943f7721509af1ead77739d1e --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/arithmetic_1dc-v0-loglikelihood @@ -0,0 +1 @@ +04c3a63a6b3c579bd3775d92b3076ba9130041d5ce7cf9244d3f86e95c804387 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/arithmetic_1dc-v0-res.json b/lm-evaluation-harness/tests/testdata/arithmetic_1dc-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..29e447d578ed11f77d962c079e9db9e3f415d801 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/arithmetic_1dc-v0-res.json @@ -0,0 +1 @@ +{"results": {"arithmetic_1dc": {"acc": 0.0, "acc_stderr": 0.0}}, "versions": {"arithmetic_1dc": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/arithmetic_2da-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/arithmetic_2da-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..fd95bb231e198d674a556bbec09b2334f1ef1a8e --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/arithmetic_2da-v0-loglikelihood @@ -0,0 +1 @@ +6ca1ca6ebd7cac4420d5005f7f35b0edbc921377f5e4f8874cc176e4fb6d79d4 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/arithmetic_2da-v0-res.json b/lm-evaluation-harness/tests/testdata/arithmetic_2da-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..874256a0b8ae0c6fe4874498ecb9e73f383f0d60 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/arithmetic_2da-v0-res.json @@ -0,0 +1 @@ +{"results": {"arithmetic_2da": {"acc": 0.0, "acc_stderr": 0.0}}, "versions": {"arithmetic_2da": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/arithmetic_2dm-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/arithmetic_2dm-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..7b7adaf86251b258f270478b8310660d56a15f4a --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/arithmetic_2dm-v0-loglikelihood @@ -0,0 +1 @@ +14ac5e510cdf82967d6827a9ca059906ee1db2e347be1b17f36403a157e73552 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/arithmetic_2dm-v0-res.json b/lm-evaluation-harness/tests/testdata/arithmetic_2dm-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..8fc5d47310794c3ec4228c51ccb05e58c90aad5c --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/arithmetic_2dm-v0-res.json @@ -0,0 +1 @@ +{"results": {"arithmetic_2dm": {"acc": 0.0, "acc_stderr": 0.0}}, "versions": {"arithmetic_2dm": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/arithmetic_2ds-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/arithmetic_2ds-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..28f32c92c67df30eb1548fd27939b45b484a4cbc --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/arithmetic_2ds-v0-loglikelihood @@ -0,0 +1 @@ +66f7ff3b40251ee38fadcbee658e309a200224356fc3efa07d0a490a2c24bfa3 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/arithmetic_2ds-v0-res.json b/lm-evaluation-harness/tests/testdata/arithmetic_2ds-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..a18e6eec6e5fc11e6a613618dddd770e96d8fdd8 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/arithmetic_2ds-v0-res.json @@ -0,0 +1 @@ +{"results": {"arithmetic_2ds": {"acc": 0.0, "acc_stderr": 0.0}}, "versions": {"arithmetic_2ds": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/arithmetic_3da-v0-res.json b/lm-evaluation-harness/tests/testdata/arithmetic_3da-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..1bbb3eb0c26b177cb739f58d8098b339278fcd84 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/arithmetic_3da-v0-res.json @@ -0,0 +1 @@ +{"results": {"arithmetic_3da": {"acc": 0.0, "acc_stderr": 0.0}}, "versions": {"arithmetic_3da": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/arithmetic_3ds-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/arithmetic_3ds-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..6bc029c520d8787ad45e3bfd5d728da3e65f15cf --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/arithmetic_3ds-v0-loglikelihood @@ -0,0 +1 @@ +d3d8bad8827d4530945a1d8b3c7589c0235bbed0bc89e7561a6fdac678f6ce5c \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/arithmetic_3ds-v0-res.json b/lm-evaluation-harness/tests/testdata/arithmetic_3ds-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..d76cc9bdf55935bc1bc4e71d35267cb58ec618ef --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/arithmetic_3ds-v0-res.json @@ -0,0 +1 @@ +{"results": {"arithmetic_3ds": {"acc": 0.0, "acc_stderr": 0.0}}, "versions": {"arithmetic_3ds": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/arithmetic_4da-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/arithmetic_4da-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..b52790c74b649b455fd90ca93cc70ad23c3d129b --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/arithmetic_4da-v0-loglikelihood @@ -0,0 +1 @@ +d3557beb8b9e5704122c2fc6362b11fbe2c3f2f3cb72aed4462b208767c40e01 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/arithmetic_4ds-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/arithmetic_4ds-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..154cf9c5946ed829ce7e2f173a2b03554fe789a1 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/arithmetic_4ds-v0-loglikelihood @@ -0,0 +1 @@ +d915830b8621e66331383bb2ae4c60acebf008e2f94741092ef4c33ea5441037 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/arithmetic_4ds-v0-res.json b/lm-evaluation-harness/tests/testdata/arithmetic_4ds-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..4321db2604d4ef8d992f587841264964acfb065f --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/arithmetic_4ds-v0-res.json @@ -0,0 +1 @@ +{"results": {"arithmetic_4ds": {"acc": 0.0, "acc_stderr": 0.0}}, "versions": {"arithmetic_4ds": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/arithmetic_5da-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/arithmetic_5da-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..a751332bc6fbae7b680f4412609dcf0695eb972c --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/arithmetic_5da-v0-loglikelihood @@ -0,0 +1 @@ +49edb1e735660631ea6cc309721e6c0b80b7106a613a6959514852ca48f1130e \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/arithmetic_5ds-v0-res.json b/lm-evaluation-harness/tests/testdata/arithmetic_5ds-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..c7773f373de3cf44b0d750454a45c4cb581a9957 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/arithmetic_5ds-v0-res.json @@ -0,0 +1 @@ +{"results": {"arithmetic_5ds": {"acc": 0.0, "acc_stderr": 0.0}}, "versions": {"arithmetic_5ds": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/blimp_principle_A_case_1-v0-res.json b/lm-evaluation-harness/tests/testdata/blimp_principle_A_case_1-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..f325c2e3e34f2d07f90e32517bf236339bd63b48 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/blimp_principle_A_case_1-v0-res.json @@ -0,0 +1 @@ +{"results": {"blimp_principle_A_case_1": {"acc": 0.485, "acc_stderr": 0.0158121796418149}}, "versions": {"blimp_principle_A_case_1": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/blimp_principle_A_domain_1-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/blimp_principle_A_domain_1-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..6b900d05f4ab0e4143324c919e684900299e9adc --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/blimp_principle_A_domain_1-v0-loglikelihood @@ -0,0 +1 @@ +290e7eddacea4ec16989af697f2ee3373fdd9aef4b452bf887184c6e2f6e7d9d \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/blimp_principle_A_reconstruction-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/blimp_principle_A_reconstruction-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..f8d1d1f87fb4347f4261920ccb2f12fdda14b7fb --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/blimp_principle_A_reconstruction-v0-loglikelihood @@ -0,0 +1 @@ +894efedfd8750d5b8de6157f9b2ed2b51b5290d3a78ea9b041fc62d34e96efbc \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/blimp_regular_plural_subject_verb_agreement_1-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/blimp_regular_plural_subject_verb_agreement_1-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..0a32ca7f971e537ab6fc6d338db3ad1c3d506f64 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/blimp_regular_plural_subject_verb_agreement_1-v0-loglikelihood @@ -0,0 +1 @@ +5bc0441f31e32443cf761bca6e961d504e1e84b15aa4e1d79e5c8ed5b4c2aa3a \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/blimp_regular_plural_subject_verb_agreement_1-v0-res.json b/lm-evaluation-harness/tests/testdata/blimp_regular_plural_subject_verb_agreement_1-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..16fed715d4effd467e798c56399f0ed4729bd49c --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/blimp_regular_plural_subject_verb_agreement_1-v0-res.json @@ -0,0 +1 @@ +{"results": {"blimp_regular_plural_subject_verb_agreement_1": {"acc": 0.485, "acc_stderr": 0.0158121796418149}}, "versions": {"blimp_regular_plural_subject_verb_agreement_1": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/blimp_regular_plural_subject_verb_agreement_2-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/blimp_regular_plural_subject_verb_agreement_2-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..4b6525a10ebb7ed53b78dc1f18553ad5896b0691 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/blimp_regular_plural_subject_verb_agreement_2-v0-loglikelihood @@ -0,0 +1 @@ +f69d9891f59872538962221fccc425b07df7cfbd83cdc546ce83e6b0e9a93f7c \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/blimp_regular_plural_subject_verb_agreement_2-v0-res.json b/lm-evaluation-harness/tests/testdata/blimp_regular_plural_subject_verb_agreement_2-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..6d64b97e20bb4688afca5e708f7fc41243ecca14 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/blimp_regular_plural_subject_verb_agreement_2-v0-res.json @@ -0,0 +1 @@ +{"results": {"blimp_regular_plural_subject_verb_agreement_2": {"acc": 0.485, "acc_stderr": 0.0158121796418149}}, "versions": {"blimp_regular_plural_subject_verb_agreement_2": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/blimp_sentential_negation_npi_licensor_present-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/blimp_sentential_negation_npi_licensor_present-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..8e254de7a73880a1880c1632e88d91fb4a9affdc --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/blimp_sentential_negation_npi_licensor_present-v0-loglikelihood @@ -0,0 +1 @@ +e6666c5657215ff4bfd646b8ee3ae6df956e71c0be9ab1c287fb1b68291dd0d1 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/blimp_sentential_negation_npi_licensor_present-v0-res.json b/lm-evaluation-harness/tests/testdata/blimp_sentential_negation_npi_licensor_present-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..4305bb313c67880a0e4ebf7827c29a2aa2df6d66 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/blimp_sentential_negation_npi_licensor_present-v0-res.json @@ -0,0 +1 @@ +{"results": {"blimp_sentential_negation_npi_licensor_present": {"acc": 0.485, "acc_stderr": 0.0158121796418149}}, "versions": {"blimp_sentential_negation_npi_licensor_present": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/blimp_sentential_negation_npi_scope-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/blimp_sentential_negation_npi_scope-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..c7aa260f9198481df3d83af52c9c16cc9e877d40 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/blimp_sentential_negation_npi_scope-v0-loglikelihood @@ -0,0 +1 @@ +32fcbd0a1c6e664af2751bad552587b5ca3911973b07f4fb2cf0a2acd3de5349 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/blimp_sentential_negation_npi_scope-v0-res.json b/lm-evaluation-harness/tests/testdata/blimp_sentential_negation_npi_scope-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..fcaf915f36cfa6a15cb5cf52f786ad96adb8eecb --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/blimp_sentential_negation_npi_scope-v0-res.json @@ -0,0 +1 @@ +{"results": {"blimp_sentential_negation_npi_scope": {"acc": 0.485, "acc_stderr": 0.0158121796418149}}, "versions": {"blimp_sentential_negation_npi_scope": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/blimp_sentential_subject_island-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/blimp_sentential_subject_island-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..6220172936ccbee00cc7d5420c30893109d366b2 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/blimp_sentential_subject_island-v0-loglikelihood @@ -0,0 +1 @@ +80f5f98fad26240de2767fe58c4b18d864df41cbfa76f06c84c3fce9f14f4833 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/blimp_sentential_subject_island-v0-res.json b/lm-evaluation-harness/tests/testdata/blimp_sentential_subject_island-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..a7f8f1825ac91b69d8ba1a50a5f87f048aeb3f78 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/blimp_sentential_subject_island-v0-res.json @@ -0,0 +1 @@ +{"results": {"blimp_sentential_subject_island": {"acc": 0.485, "acc_stderr": 0.0158121796418149}}, "versions": {"blimp_sentential_subject_island": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/blimp_superlative_quantifiers_1-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/blimp_superlative_quantifiers_1-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..b7d2819cb3b61b90bd5efee98e890b486fc02f39 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/blimp_superlative_quantifiers_1-v0-loglikelihood @@ -0,0 +1 @@ +8a01f6a5ea87a01c0c9b0c7b3bc4de4711bf0ff050976976651182b9ed34a0d4 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/blimp_superlative_quantifiers_1-v0-res.json b/lm-evaluation-harness/tests/testdata/blimp_superlative_quantifiers_1-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..b69d445f3c257608fd5be46aa74bd53cd598042c --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/blimp_superlative_quantifiers_1-v0-res.json @@ -0,0 +1 @@ +{"results": {"blimp_superlative_quantifiers_1": {"acc": 0.485, "acc_stderr": 0.0158121796418149}}, "versions": {"blimp_superlative_quantifiers_1": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/blimp_superlative_quantifiers_2-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/blimp_superlative_quantifiers_2-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..4a8317f0b3ac61c3e677a5caa03bd47223a3fb7b --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/blimp_superlative_quantifiers_2-v0-loglikelihood @@ -0,0 +1 @@ +59c20ff0f632cf42afc74ecc682cf92e5e740417b01e6cf9a610a3bc544d2ea5 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/blimp_superlative_quantifiers_2-v0-res.json b/lm-evaluation-harness/tests/testdata/blimp_superlative_quantifiers_2-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..2733d251cf90f264f28db48a2b17b520e528f2c7 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/blimp_superlative_quantifiers_2-v0-res.json @@ -0,0 +1 @@ +{"results": {"blimp_superlative_quantifiers_2": {"acc": 0.485, "acc_stderr": 0.0158121796418149}}, "versions": {"blimp_superlative_quantifiers_2": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/blimp_tough_vs_raising_1-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/blimp_tough_vs_raising_1-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..a26cb174a06e1941ae79e137161d85c4f5814838 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/blimp_tough_vs_raising_1-v0-loglikelihood @@ -0,0 +1 @@ +973fe56534fdef1207f0fc08dd09a210304c55f33c6cbb17552754bf54f11c86 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/blimp_tough_vs_raising_1-v0-res.json b/lm-evaluation-harness/tests/testdata/blimp_tough_vs_raising_1-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..44ea10c1380c3dccdbc8d2ad6a2d84e716e81773 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/blimp_tough_vs_raising_1-v0-res.json @@ -0,0 +1 @@ +{"results": {"blimp_tough_vs_raising_1": {"acc": 0.485, "acc_stderr": 0.0158121796418149}}, "versions": {"blimp_tough_vs_raising_1": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/blimp_tough_vs_raising_2-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/blimp_tough_vs_raising_2-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..3b0f9763529ee45a97ab0abdfd18efc9fe991241 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/blimp_tough_vs_raising_2-v0-loglikelihood @@ -0,0 +1 @@ +d255a10a34f14d77d9526604a17b0f6747d32f62fc2e3a09e9ab10054535fd45 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/blimp_tough_vs_raising_2-v0-res.json b/lm-evaluation-harness/tests/testdata/blimp_tough_vs_raising_2-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..c9b8c7d06179f5427a99dda5e6b24245e2ea0dbb --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/blimp_tough_vs_raising_2-v0-res.json @@ -0,0 +1 @@ +{"results": {"blimp_tough_vs_raising_2": {"acc": 0.485, "acc_stderr": 0.0158121796418149}}, "versions": {"blimp_tough_vs_raising_2": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/blimp_transitive-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/blimp_transitive-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..98156dcf1ea33db946094d1e9d47c979f158b8b2 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/blimp_transitive-v0-loglikelihood @@ -0,0 +1 @@ +d0d47fe40a7ee558ba782edbc4f49f7d9123c8472a36decc97f8ab142b45b9d8 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/blimp_transitive-v0-res.json b/lm-evaluation-harness/tests/testdata/blimp_transitive-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..d2c99ab803288212934142c2507a8c316695a34b --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/blimp_transitive-v0-res.json @@ -0,0 +1 @@ +{"results": {"blimp_transitive": {"acc": 0.485, "acc_stderr": 0.0158121796418149}}, "versions": {"blimp_transitive": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/blimp_wh_island-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/blimp_wh_island-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..d27f1316dc96be401dee9392f973e9bbd799a409 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/blimp_wh_island-v0-loglikelihood @@ -0,0 +1 @@ +91a9e4b60b0f3572a7fdbd7648d0e69f36e5eb34db715315b0082558d7ed8b65 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/blimp_wh_island-v0-res.json b/lm-evaluation-harness/tests/testdata/blimp_wh_island-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..1d50683774ecacf772eaf6287328994d4abc0a98 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/blimp_wh_island-v0-res.json @@ -0,0 +1 @@ +{"results": {"blimp_wh_island": {"acc": 0.485, "acc_stderr": 0.0158121796418149}}, "versions": {"blimp_wh_island": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/blimp_wh_questions_object_gap-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/blimp_wh_questions_object_gap-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..c3e6af12f2da0a1857c0f0456bf4052d5558329e --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/blimp_wh_questions_object_gap-v0-loglikelihood @@ -0,0 +1 @@ +4d4aaa0274ccd485ff8430ed61b8f83806febe18c16616c7d050f637a0463eba \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/blimp_wh_questions_object_gap-v0-res.json b/lm-evaluation-harness/tests/testdata/blimp_wh_questions_object_gap-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..60228b79185dd96e5e71a1c2f85ade32348d9f10 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/blimp_wh_questions_object_gap-v0-res.json @@ -0,0 +1 @@ +{"results": {"blimp_wh_questions_object_gap": {"acc": 0.485, "acc_stderr": 0.0158121796418149}}, "versions": {"blimp_wh_questions_object_gap": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/blimp_wh_questions_subject_gap-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/blimp_wh_questions_subject_gap-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..1a88f8fa87a86cbff20d0def9955059e9cd73861 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/blimp_wh_questions_subject_gap-v0-loglikelihood @@ -0,0 +1 @@ +d5486ffcc075cad4302e37ece9bbf5b2063c0b5a48e76c8e1dd365e22a5a48fc \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/blimp_wh_questions_subject_gap-v0-res.json b/lm-evaluation-harness/tests/testdata/blimp_wh_questions_subject_gap-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..4b21da71d54c0a8fe09f204dd4a78f0841c6ae85 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/blimp_wh_questions_subject_gap-v0-res.json @@ -0,0 +1 @@ +{"results": {"blimp_wh_questions_subject_gap": {"acc": 0.485, "acc_stderr": 0.0158121796418149}}, "versions": {"blimp_wh_questions_subject_gap": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/blimp_wh_questions_subject_gap_long_distance-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/blimp_wh_questions_subject_gap_long_distance-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..f83ed1fb7413ddccae66c32078a9a5f7b19eb03e --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/blimp_wh_questions_subject_gap_long_distance-v0-loglikelihood @@ -0,0 +1 @@ +37483dfda688b62ad27161c9fc1e1e7710c5a6e6a7cd3474df119bcafd30e97f \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/blimp_wh_questions_subject_gap_long_distance-v0-res.json b/lm-evaluation-harness/tests/testdata/blimp_wh_questions_subject_gap_long_distance-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..fe6bbf95e5406ad38d4894bf5d4609beeaa05f9a --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/blimp_wh_questions_subject_gap_long_distance-v0-res.json @@ -0,0 +1 @@ +{"results": {"blimp_wh_questions_subject_gap_long_distance": {"acc": 0.485, "acc_stderr": 0.0158121796418149}}, "versions": {"blimp_wh_questions_subject_gap_long_distance": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/blimp_wh_vs_that_no_gap-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/blimp_wh_vs_that_no_gap-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..5f40ea63f1b31bfc83b5aa0385051fbcbc3574d8 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/blimp_wh_vs_that_no_gap-v0-loglikelihood @@ -0,0 +1 @@ +d1d3e439b2020ef5ed232bfebbcc9634adc5117e9eb61e38fdbbe2c8ea128d54 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/blimp_wh_vs_that_no_gap-v0-res.json b/lm-evaluation-harness/tests/testdata/blimp_wh_vs_that_no_gap-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..dfd3f66b77cb52234d967a827a3c6dffc706e5aa --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/blimp_wh_vs_that_no_gap-v0-res.json @@ -0,0 +1 @@ +{"results": {"blimp_wh_vs_that_no_gap": {"acc": 0.485, "acc_stderr": 0.0158121796418149}}, "versions": {"blimp_wh_vs_that_no_gap": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/blimp_wh_vs_that_no_gap_long_distance-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/blimp_wh_vs_that_no_gap_long_distance-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..13359ac3d2092bb8d38d44f17a125124c034d317 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/blimp_wh_vs_that_no_gap_long_distance-v0-loglikelihood @@ -0,0 +1 @@ +a142cc2a6fcd93230b650927b07367cad957b8f3f42cb4072151da53dea301df \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/blimp_wh_vs_that_no_gap_long_distance-v0-res.json b/lm-evaluation-harness/tests/testdata/blimp_wh_vs_that_no_gap_long_distance-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..de9e8007180f265cb7b2aed51e277b93fded9ce6 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/blimp_wh_vs_that_no_gap_long_distance-v0-res.json @@ -0,0 +1 @@ +{"results": {"blimp_wh_vs_that_no_gap_long_distance": {"acc": 0.485, "acc_stderr": 0.0158121796418149}}, "versions": {"blimp_wh_vs_that_no_gap_long_distance": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/blimp_wh_vs_that_with_gap-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/blimp_wh_vs_that_with_gap-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..4c15f2283eb93c5ab4b9cdbddf3e91117211918d --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/blimp_wh_vs_that_with_gap-v0-loglikelihood @@ -0,0 +1 @@ +d41a9b85e4c31e445bf9b46b8642df02203ccc02b4a9b254bf76066d5c54b4b7 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/blimp_wh_vs_that_with_gap-v0-res.json b/lm-evaluation-harness/tests/testdata/blimp_wh_vs_that_with_gap-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..14befd4ab6450dbb2147d66e5458981756bfc25b --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/blimp_wh_vs_that_with_gap-v0-res.json @@ -0,0 +1 @@ +{"results": {"blimp_wh_vs_that_with_gap": {"acc": 0.485, "acc_stderr": 0.0158121796418149}}, "versions": {"blimp_wh_vs_that_with_gap": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/blimp_wh_vs_that_with_gap_long_distance-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/blimp_wh_vs_that_with_gap_long_distance-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..34b959139635a241b0fe814ce2ae7240c32a7c1c --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/blimp_wh_vs_that_with_gap_long_distance-v0-loglikelihood @@ -0,0 +1 @@ +eed67491bdf493a1dad8f1d9766bc7bd0e79946365b833c0f7eb81ac998e3dca \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/blimp_wh_vs_that_with_gap_long_distance-v0-res.json b/lm-evaluation-harness/tests/testdata/blimp_wh_vs_that_with_gap_long_distance-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..95a2c0c7e115167e44288a57dc38ea1d40274c87 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/blimp_wh_vs_that_with_gap_long_distance-v0-res.json @@ -0,0 +1 @@ +{"results": {"blimp_wh_vs_that_with_gap_long_distance": {"acc": 0.485, "acc_stderr": 0.0158121796418149}}, "versions": {"blimp_wh_vs_that_with_gap_long_distance": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/boolq-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/boolq-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..14c1bf5f5ee1300b8652f6a73185badea754ec73 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/boolq-v0-loglikelihood @@ -0,0 +1 @@ +de5aa6f77a2e0fd050b9c272f10c4d5d5581e4f75ffa60926f79e60ae1738960 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/boolq-v0-res.json b/lm-evaluation-harness/tests/testdata/boolq-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..2b459d8b28901b531dc0068425a4583747ac552d --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/boolq-v0-res.json @@ -0,0 +1 @@ +{"results": {"boolq": {"acc": 0.5048929663608562, "acc_stderr": 0.00874463623355505}}, "versions": {"boolq": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/boolq-v1-loglikelihood b/lm-evaluation-harness/tests/testdata/boolq-v1-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..7811121c9fda0c7ec33c2c36639c8ed8febccb05 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/boolq-v1-loglikelihood @@ -0,0 +1 @@ +6577e0d88572772ef08e64f624c0e3df0953286ae1f118ccef15623b59ffeabf \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/boolq-v1-res.json b/lm-evaluation-harness/tests/testdata/boolq-v1-res.json new file mode 100644 index 0000000000000000000000000000000000000000..291b9f122d0219c93c941daeb9ae362c439bb4e0 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/boolq-v1-res.json @@ -0,0 +1 @@ +{"results": {"boolq": {"acc": 0.5048929663608562, "acc_stderr": 0.00874463623355505}}, "versions": {"boolq": 1}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/cb-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/cb-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..6fa6f6dae6c806be8a5cad8416df6766f22ae475 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/cb-v0-loglikelihood @@ -0,0 +1 @@ +ec3b1bbb9561e39c43c6f77a23b4060b15c606141c5346e3d0791b3e92aaa5d0 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/cb-v0-res.json b/lm-evaluation-harness/tests/testdata/cb-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..ba386fd6c7e67c5048d2f4a4240e1b308dca7db5 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/cb-v0-res.json @@ -0,0 +1 @@ +{"results": {"cb": {"acc": 0.3392857142857143, "acc_stderr": 0.06384226561930825, "f1": 0.2819143819143819}}, "versions": {"cb": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/cb-v1-loglikelihood b/lm-evaluation-harness/tests/testdata/cb-v1-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..ad7e928fe6a3d79857c3c076c6459d8b6c31897c --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/cb-v1-loglikelihood @@ -0,0 +1 @@ +77b11f4348eb8a7f57faf95c531fda01ab4bf0e729f91a82451ed8e71ec8e66d \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/cb-v1-res.json b/lm-evaluation-harness/tests/testdata/cb-v1-res.json new file mode 100644 index 0000000000000000000000000000000000000000..1cff410b2c35a16b457d163d95ac7cbd8eb704e2 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/cb-v1-res.json @@ -0,0 +1 @@ +{"results": {"cb": {"acc": 0.3392857142857143, "acc_stderr": 0.06384226561930825, "f1": 0.2819143819143819}}, "versions": {"cb": 1}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/cola-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/cola-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..45737909e7c21c528a647e91cceca3d2534869fc --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/cola-v0-loglikelihood @@ -0,0 +1 @@ +e8635578ed8ee70b707a666d35e468b9321db24470f80c92080651e2bfa01751 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/cola-v0-res.json b/lm-evaluation-harness/tests/testdata/cola-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..462e5d9401318226da067adcc39b27a09157a127 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/cola-v0-res.json @@ -0,0 +1 @@ +{"results": {"cola": {"mcc": -0.04538802810223175, "mcc_stderr": 0.023100371589225246}}, "versions": {"cola": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/copa-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/copa-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..ebe4c6512a5a4befba815e4ab3b52a3732600607 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/copa-v0-loglikelihood @@ -0,0 +1 @@ +66276b9045b5300cba4b81340db06f674f031fa0b8883714ad0d03be464cd799 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/copa-v0-res.json b/lm-evaluation-harness/tests/testdata/copa-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..9a537ec768e7311cd4ef3fafcfde63cf9ff42f59 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/copa-v0-res.json @@ -0,0 +1 @@ +{"results": {"copa": {"acc": 0.48, "acc_stderr": 0.050211673156867795}}, "versions": {"copa": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/coqa-v0-greedy_until b/lm-evaluation-harness/tests/testdata/coqa-v0-greedy_until new file mode 100644 index 0000000000000000000000000000000000000000..c1a9e165a7e42191745e38b1dd8d6b9e2fe609cb --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/coqa-v0-greedy_until @@ -0,0 +1 @@ +4a8605d5deed0423ec095700251ed93325b45d320aca35d4ce1e94702094435e \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/coqa-v0-res.json b/lm-evaluation-harness/tests/testdata/coqa-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..9ca8024e3ba0fb80420952bceaf01d85c42b0fcc --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/coqa-v0-res.json @@ -0,0 +1 @@ +{"results": {"coqa": {"em": 0.0, "em_stderr": 0.0, "f1": 0.0, "f1_stderr": 0.0}}, "versions": {"coqa": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/coqa-v1-greedy_until b/lm-evaluation-harness/tests/testdata/coqa-v1-greedy_until new file mode 100644 index 0000000000000000000000000000000000000000..f6e3f64b18a1d7d3ec2702d115c694bbe62cc8ef --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/coqa-v1-greedy_until @@ -0,0 +1 @@ +57581470b921435d40da97872bb1cfda6ecf963ccc4b0240a3b04e3fea8c8e3a \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/coqa-v1-res.json b/lm-evaluation-harness/tests/testdata/coqa-v1-res.json new file mode 100644 index 0000000000000000000000000000000000000000..7941ad62997cb5129be9390a727352b689f807ae --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/coqa-v1-res.json @@ -0,0 +1 @@ +{"results": {"coqa": {"em": 0.0, "em_stderr": 0.0, "f1": 0.0, "f1_stderr": 0.0}}, "versions": {"coqa": 1}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/crows_pairs_english-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/crows_pairs_english-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..63749433f1703a4c81965e6c04fec04177631bae --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/crows_pairs_english-v0-loglikelihood @@ -0,0 +1 @@ +ee3ce1ddb8071d4189e5b06e7f3c618a434221ac52935d0f434c4d183f01458a \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/crows_pairs_english-v0-res.json b/lm-evaluation-harness/tests/testdata/crows_pairs_english-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..c4210f5f11540d44476cdf99252e9268ca85a6e0 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/crows_pairs_english-v0-res.json @@ -0,0 +1 @@ +{"results": {"crows_pairs_english": {"likelihood_difference": 0.3367363060632734, "likelihood_difference_stderr": 0.005827747024053628, "pct_stereotype": 0.5062611806797853, "pct_stereotype_stderr": 0.012212341600228745}}, "versions": {"crows_pairs_english": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/crows_pairs_english_age-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/crows_pairs_english_age-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..598d2cce10cc3ecefb6eb8d1deb74801e25b11af --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/crows_pairs_english_age-v0-loglikelihood @@ -0,0 +1 @@ +de74d2ac7f926f2f486c045d84aae8f71711102f9d77b31f758fd148810d13d3 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/crows_pairs_english_age-v0-res.json b/lm-evaluation-harness/tests/testdata/crows_pairs_english_age-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..5dad8bf864709209d905dadb52930eaf43ff3eb0 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/crows_pairs_english_age-v0-res.json @@ -0,0 +1 @@ +{"results": {"crows_pairs_english_age": {"likelihood_difference": 0.3160680928470684, "likelihood_difference_stderr": 0.02397758321605678, "pct_stereotype": 0.43956043956043955, "pct_stereotype_stderr": 0.05231815698566189}}, "versions": {"crows_pairs_english_age": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/crows_pairs_english_autre-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/crows_pairs_english_autre-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..ab0d9a4db42a5e4da196834b40457a95bf9a9129 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/crows_pairs_english_autre-v0-loglikelihood @@ -0,0 +1 @@ +a197ccc8538231404a8e43f5ed0fbbfb2c317b4da337f6e7aa9642131aeb426a \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/crows_pairs_english_autre-v0-res.json b/lm-evaluation-harness/tests/testdata/crows_pairs_english_autre-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..dbe264794f6009bd604d2d55928e1958c74ae35a --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/crows_pairs_english_autre-v0-res.json @@ -0,0 +1 @@ +{"results": {"crows_pairs_english_autre": {"likelihood_difference": 0.3424336593343321, "likelihood_difference_stderr": 0.08588068996335849, "pct_stereotype": 0.2727272727272727, "pct_stereotype_stderr": 0.14083575804390605}}, "versions": {"crows_pairs_english_autre": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/crows_pairs_english_disability-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/crows_pairs_english_disability-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..50c7b025631010289ee73762c8f493d8888122d3 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/crows_pairs_english_disability-v0-loglikelihood @@ -0,0 +1 @@ +90c1bcfdeec0ff51d891ee8cf00ae2a5ec61bab6739faea9865809b8ffed2cdb \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/crows_pairs_english_disability-v0-res.json b/lm-evaluation-harness/tests/testdata/crows_pairs_english_disability-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..14510a13a1c390adfbb9c73149b88e5b8a2c4f64 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/crows_pairs_english_disability-v0-res.json @@ -0,0 +1 @@ +{"results": {"crows_pairs_english_disability": {"likelihood_difference": 0.3148684792547637, "likelihood_difference_stderr": 0.02800803147051987, "pct_stereotype": 0.36923076923076925, "pct_stereotype_stderr": 0.06032456592830047}}, "versions": {"crows_pairs_english_disability": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/crows_pairs_english_gender-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/crows_pairs_english_gender-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..44a4c513e5ba88fe7ed54dcb35021b709bc407e2 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/crows_pairs_english_gender-v0-loglikelihood @@ -0,0 +1 @@ +2bf62b7cc678f64ffad4a6e6715ff76a2b984bfe8d1165da4b76b3b4dfafb2f9 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/crows_pairs_english_gender-v0-res.json b/lm-evaluation-harness/tests/testdata/crows_pairs_english_gender-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..c24fb9dd6dfb9c494474fc08011d6d86ef18f5ef --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/crows_pairs_english_gender-v0-res.json @@ -0,0 +1 @@ +{"results": {"crows_pairs_english_gender": {"likelihood_difference": 0.3361377482385407, "likelihood_difference_stderr": 0.012853081126751691, "pct_stereotype": 0.478125, "pct_stereotype_stderr": 0.027967820983765136}}, "versions": {"crows_pairs_english_gender": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/crows_pairs_english_nationality-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/crows_pairs_english_nationality-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..1186c252981559fc1e9859252f82aaea27310c4f --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/crows_pairs_english_nationality-v0-loglikelihood @@ -0,0 +1 @@ +b85bc849811ccfa9971a6ee3fca7342752c314c0cb6f126e10d9ec4d0450c541 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/crows_pairs_english_nationality-v0-res.json b/lm-evaluation-harness/tests/testdata/crows_pairs_english_nationality-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..5fd526ccc1c07111d2cceef633ccb72b0d65387b --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/crows_pairs_english_nationality-v0-res.json @@ -0,0 +1 @@ +{"results": {"crows_pairs_english_nationality": {"likelihood_difference": 0.3383027778174895, "likelihood_difference_stderr": 0.015957585374543233, "pct_stereotype": 0.4675925925925926, "pct_stereotype_stderr": 0.03402801581358966}}, "versions": {"crows_pairs_english_nationality": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/crows_pairs_english_physical_appearance-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/crows_pairs_english_physical_appearance-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..fedfdac52d966f6edcdb229456858da1959b24d1 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/crows_pairs_english_physical_appearance-v0-loglikelihood @@ -0,0 +1 @@ +d1823f5038afafa7a5338e42531720480c8ccf4e177789526caf294d52d56e89 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/crows_pairs_english_physical_appearance-v0-res.json b/lm-evaluation-harness/tests/testdata/crows_pairs_english_physical_appearance-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..4e5ce5b3e7d4df6366fc2cb0219a24b92f1fabed --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/crows_pairs_english_physical_appearance-v0-res.json @@ -0,0 +1 @@ +{"results": {"crows_pairs_english_physical_appearance": {"likelihood_difference": 0.3221673223187262, "likelihood_difference_stderr": 0.026978346460100555, "pct_stereotype": 0.4027777777777778, "pct_stereotype_stderr": 0.05820650942569533}}, "versions": {"crows_pairs_english_physical_appearance": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/crows_pairs_english_race_color-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/crows_pairs_english_race_color-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..9feec03298368b126f4c7361084fb894b8170ffd --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/crows_pairs_english_race_color-v0-loglikelihood @@ -0,0 +1 @@ +0a750596d77cd96502dc414ff699a399b1b91c2078adeec1d3dd982b3d591089 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/crows_pairs_english_race_color-v0-res.json b/lm-evaluation-harness/tests/testdata/crows_pairs_english_race_color-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..75d356522edf08f93f03d3ba37ed323d39f5b35e --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/crows_pairs_english_race_color-v0-res.json @@ -0,0 +1 @@ +{"results": {"crows_pairs_english_race_color": {"likelihood_difference": 0.3322827903840805, "likelihood_difference_stderr": 0.01019838186372816, "pct_stereotype": 0.4822834645669291, "pct_stereotype_stderr": 0.022191835500120254}}, "versions": {"crows_pairs_english_race_color": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/crows_pairs_english_religion-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/crows_pairs_english_religion-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..b56bc901ca48380f5a188f9c18ef12ba0abe49ca --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/crows_pairs_english_religion-v0-loglikelihood @@ -0,0 +1 @@ +2ed57377174adaf0fb30037eb055eafdd02cd46e57bc32066d5fecd90a14b6e1 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/crows_pairs_english_religion-v0-res.json b/lm-evaluation-harness/tests/testdata/crows_pairs_english_religion-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..670f2d2cffeac37f0510e17d7195a0a68700d4fe --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/crows_pairs_english_religion-v0-res.json @@ -0,0 +1 @@ +{"results": {"crows_pairs_english_religion": {"likelihood_difference": 0.32170622542430666, "likelihood_difference_stderr": 0.022101541392310232, "pct_stereotype": 0.43243243243243246, "pct_stereotype_stderr": 0.04723583229758394}}, "versions": {"crows_pairs_english_religion": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/crows_pairs_english_sexual_orientation-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/crows_pairs_english_sexual_orientation-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..0a58b730c1e43271ba9d287c6b645ab97d10a560 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/crows_pairs_english_sexual_orientation-v0-loglikelihood @@ -0,0 +1 @@ +e754a309296b157677dfba6e6feef983d1ce38dd0169ae726265621a7b573163 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/crows_pairs_english_sexual_orientation-v0-res.json b/lm-evaluation-harness/tests/testdata/crows_pairs_english_sexual_orientation-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..9a93b9add705c62cd228fd21a89ea670022189ab --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/crows_pairs_english_sexual_orientation-v0-res.json @@ -0,0 +1 @@ +{"results": {"crows_pairs_english_sexual_orientation": {"likelihood_difference": 0.31947594049467243, "likelihood_difference_stderr": 0.024404952720497735, "pct_stereotype": 0.43010752688172044, "pct_stereotype_stderr": 0.051616798980291805}}, "versions": {"crows_pairs_english_sexual_orientation": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/crows_pairs_english_socioeconomic-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/crows_pairs_english_socioeconomic-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..32065ff76227a91aa5631f95b66ff1ce19490800 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/crows_pairs_english_socioeconomic-v0-loglikelihood @@ -0,0 +1 @@ +c309eabfd247a702e32efc4e08211f9a72693d38995be5dd444d497b476396bd \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/crows_pairs_english_socioeconomic-v0-res.json b/lm-evaluation-harness/tests/testdata/crows_pairs_english_socioeconomic-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..89bd7338ada6ff7ef485492c5656342881b70600 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/crows_pairs_english_socioeconomic-v0-res.json @@ -0,0 +1 @@ +{"results": {"crows_pairs_english_socioeconomic": {"likelihood_difference": 0.3424577735757881, "likelihood_difference_stderr": 0.017459994170011896, "pct_stereotype": 0.46842105263157896, "pct_stereotype_stderr": 0.036297038088316094}}, "versions": {"crows_pairs_english_socioeconomic": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/crows_pairs_french-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/crows_pairs_french-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..d0fc86b66760ef71b6791ffeb9b9061e4cb49720 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/crows_pairs_french-v0-loglikelihood @@ -0,0 +1 @@ +4fb61dcf4d2c59d6470b297a01d5f429ee442864e225e1760fbf191b2a0901cd \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/crows_pairs_french-v0-res.json b/lm-evaluation-harness/tests/testdata/crows_pairs_french-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..77195255653eaebf9f1d542df02b9720c1f37df8 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/crows_pairs_french-v0-res.json @@ -0,0 +1 @@ +{"results": {"crows_pairs_french": {"likelihood_difference": 0.3367363060632734, "likelihood_difference_stderr": 0.005827747024053628, "pct_stereotype": 0.5062611806797853, "pct_stereotype_stderr": 0.012212341600228745}}, "versions": {"crows_pairs_french": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/crows_pairs_french_age-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/crows_pairs_french_age-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..dbec353c35db547d54e918c718164a0788abc569 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/crows_pairs_french_age-v0-loglikelihood @@ -0,0 +1 @@ +b14a5769f415a234abe89063a1b546aa4a990c84217e5d4a697874cd7f85af35 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/crows_pairs_french_age-v0-res.json b/lm-evaluation-harness/tests/testdata/crows_pairs_french_age-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..4bd87f68c37946bcb26e2a989e98a79251a8361c --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/crows_pairs_french_age-v0-res.json @@ -0,0 +1 @@ +{"results": {"crows_pairs_french_age": {"likelihood_difference": 0.31896094607685194, "likelihood_difference_stderr": 0.024068391933540753, "pct_stereotype": 0.4444444444444444, "pct_stereotype_stderr": 0.05267171812666418}}, "versions": {"crows_pairs_french_age": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/crows_pairs_french_autre-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/crows_pairs_french_autre-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..3900f561993a333909d46e7a4fc18906c9b69721 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/crows_pairs_french_autre-v0-loglikelihood @@ -0,0 +1 @@ +f145ad5086da0bf8c76f0730258529fa243efe32b7ab792d3c4716284b4b5495 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/crows_pairs_french_autre-v0-res.json b/lm-evaluation-harness/tests/testdata/crows_pairs_french_autre-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..44d8ff96e413cf6eb458a896d47321a0f3996b70 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/crows_pairs_french_autre-v0-res.json @@ -0,0 +1 @@ +{"results": {"crows_pairs_french_autre": {"likelihood_difference": 0.3517045997290783, "likelihood_difference_stderr": 0.07647821858130377, "pct_stereotype": 0.23076923076923078, "pct_stereotype_stderr": 0.12162606385262997}}, "versions": {"crows_pairs_french_autre": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/crows_pairs_french_disability-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/crows_pairs_french_disability-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..9cc4d2bb8012080bb2030e494eebd97e945b203c --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/crows_pairs_french_disability-v0-loglikelihood @@ -0,0 +1 @@ +fa1e5fc7492a66c9a90765e605003c38408347617db5ecf36706f1d374af5d42 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/crows_pairs_french_disability-v0-res.json b/lm-evaluation-harness/tests/testdata/crows_pairs_french_disability-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..cb2b8b79ac3d37913992a56e688ea80d24c0af9e --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/crows_pairs_french_disability-v0-res.json @@ -0,0 +1 @@ +{"results": {"crows_pairs_french_disability": {"likelihood_difference": 0.31387939561315326, "likelihood_difference_stderr": 0.027598132299657168, "pct_stereotype": 0.36363636363636365, "pct_stereotype_stderr": 0.05966637484671758}}, "versions": {"crows_pairs_french_disability": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/crows_pairs_french_gender-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/crows_pairs_french_gender-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..c1713a5a881657c9ae4417f6adcf7480491a2915 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/crows_pairs_french_gender-v0-loglikelihood @@ -0,0 +1 @@ +010b8404655911c86555616da23afffce9dc3981e1acbbfdb022d9c474430209 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/crows_pairs_french_gender-v0-res.json b/lm-evaluation-harness/tests/testdata/crows_pairs_french_gender-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..bdb363e75dc8006cd39e237392b2cf589741fb46 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/crows_pairs_french_gender-v0-res.json @@ -0,0 +1 @@ +{"results": {"crows_pairs_french_gender": {"likelihood_difference": 0.3364019171359413, "likelihood_difference_stderr": 0.012815700745990895, "pct_stereotype": 0.4766355140186916, "pct_stereotype_stderr": 0.027920316348204986}}, "versions": {"crows_pairs_french_gender": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/crows_pairs_french_nationality-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/crows_pairs_french_nationality-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..e6e282414b02b032bd5b879775686c24e731fd9d --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/crows_pairs_french_nationality-v0-loglikelihood @@ -0,0 +1 @@ +146eb60c8796fe3f25307a6776337f0b077b58ce02edec64c99df4b906c19b9f \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/crows_pairs_french_nationality-v0-res.json b/lm-evaluation-harness/tests/testdata/crows_pairs_french_nationality-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..f9dd321f7f3c9525491145df99fb4f7658be8065 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/crows_pairs_french_nationality-v0-res.json @@ -0,0 +1 @@ +{"results": {"crows_pairs_french_nationality": {"likelihood_difference": 0.33534193269044926, "likelihood_difference_stderr": 0.01429836309463257, "pct_stereotype": 0.4743083003952569, "pct_stereotype_stderr": 0.031455431847992904}}, "versions": {"crows_pairs_french_nationality": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/crows_pairs_french_physical_appearance-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/crows_pairs_french_physical_appearance-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..167b5e3ba055d1d67ca70e4f9cd3879f6b40b179 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/crows_pairs_french_physical_appearance-v0-loglikelihood @@ -0,0 +1 @@ +ea61eaad64e9292790d4bbef955ffeebed7a595de098bc5ac726a6e51f27f9af \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/crows_pairs_french_physical_appearance-v0-res.json b/lm-evaluation-harness/tests/testdata/crows_pairs_french_physical_appearance-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..eea3efa006503d2062660ae0e0625c85b4196899 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/crows_pairs_french_physical_appearance-v0-res.json @@ -0,0 +1 @@ +{"results": {"crows_pairs_french_physical_appearance": {"likelihood_difference": 0.3221673223187262, "likelihood_difference_stderr": 0.026978346460100555, "pct_stereotype": 0.4027777777777778, "pct_stereotype_stderr": 0.05820650942569533}}, "versions": {"crows_pairs_french_physical_appearance": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/crows_pairs_french_race_color-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/crows_pairs_french_race_color-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..16127e96ad2c7051d8daf0bc0ad5114a6a07eb69 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/crows_pairs_french_race_color-v0-loglikelihood @@ -0,0 +1 @@ +6f9119026abff33c5c882d6172e092e806a8b21bd86864022978b1961839350f \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/crows_pairs_french_race_color-v0-res.json b/lm-evaluation-harness/tests/testdata/crows_pairs_french_race_color-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..bdb9d9c6aff73eac1def51836e15733ad940835c --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/crows_pairs_french_race_color-v0-res.json @@ -0,0 +1 @@ +{"results": {"crows_pairs_french_race_color": {"likelihood_difference": 0.33233909422443764, "likelihood_difference_stderr": 0.010623405969915857, "pct_stereotype": 0.4782608695652174, "pct_stereotype_stderr": 0.023315932363473738}}, "versions": {"crows_pairs_french_race_color": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/crows_pairs_french_religion-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/crows_pairs_french_religion-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..b31daf0e281664ab74ae88a9edd6bb1029f28d57 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/crows_pairs_french_religion-v0-loglikelihood @@ -0,0 +1 @@ +8af6445eeb634dad5f0723e40615afe993e1e3f129a4f314fe4117e633c2efd3 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/crows_pairs_french_religion-v0-res.json b/lm-evaluation-harness/tests/testdata/crows_pairs_french_religion-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..990eab593f8a175be48d44c7318eeb968aab2921 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/crows_pairs_french_religion-v0-res.json @@ -0,0 +1 @@ +{"results": {"crows_pairs_french_religion": {"likelihood_difference": 0.32691651640972225, "likelihood_difference_stderr": 0.021833493193249474, "pct_stereotype": 0.45217391304347826, "pct_stereotype_stderr": 0.046614569799583463}}, "versions": {"crows_pairs_french_religion": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/crows_pairs_french_sexual_orientation-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/crows_pairs_french_sexual_orientation-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..0336c1ddc64ef089490495a817922f3e7c9bdc73 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/crows_pairs_french_sexual_orientation-v0-loglikelihood @@ -0,0 +1 @@ +2ce823fdb93d325aa8fb40db5d335b093b4b69792763532d940a752440ee3a76 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/crows_pairs_french_sexual_orientation-v0-res.json b/lm-evaluation-harness/tests/testdata/crows_pairs_french_sexual_orientation-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..5bb8a4336d89c12896186dc53f0bdd7f480c8df0 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/crows_pairs_french_sexual_orientation-v0-res.json @@ -0,0 +1 @@ +{"results": {"crows_pairs_french_sexual_orientation": {"likelihood_difference": 0.3160680928470684, "likelihood_difference_stderr": 0.02397758321605678, "pct_stereotype": 0.43956043956043955, "pct_stereotype_stderr": 0.05231815698566189}}, "versions": {"crows_pairs_french_sexual_orientation": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/crows_pairs_french_socioeconomic-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/crows_pairs_french_socioeconomic-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..2f6455aec029fea8d7ee8fa866e9f7779ac99914 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/crows_pairs_french_socioeconomic-v0-loglikelihood @@ -0,0 +1 @@ +8ba0a525c65f795c99f6416e70c998e75e4b6cc43bf9a4bd7ccacd3c3591e9cb \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/crows_pairs_french_socioeconomic-v0-res.json b/lm-evaluation-harness/tests/testdata/crows_pairs_french_socioeconomic-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..7372018798d522cdfda7e458f1d608f1a3c13169 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/crows_pairs_french_socioeconomic-v0-res.json @@ -0,0 +1 @@ +{"results": {"crows_pairs_french_socioeconomic": {"likelihood_difference": 0.3394681494647815, "likelihood_difference_stderr": 0.01702488895584347, "pct_stereotype": 0.4642857142857143, "pct_stereotype_stderr": 0.035714285714285705}}, "versions": {"crows_pairs_french_socioeconomic": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/cycle_letters-v0-greedy_until b/lm-evaluation-harness/tests/testdata/cycle_letters-v0-greedy_until new file mode 100644 index 0000000000000000000000000000000000000000..9068a24ef5af549a13fe5b4362c2b5afc741bd29 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/cycle_letters-v0-greedy_until @@ -0,0 +1 @@ +eb23f7d5de7528eefd8ed5f8054c402ff947319cccfef7195995946f99389201 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/cycle_letters-v0-res.json b/lm-evaluation-harness/tests/testdata/cycle_letters-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..5b05a9430e90ec2ce0ddcb49a243be9479d3fad1 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/cycle_letters-v0-res.json @@ -0,0 +1 @@ +{"results": {"cycle_letters": {"acc": 0.0, "acc_stderr": 0.0}}, "versions": {"cycle_letters": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/drop-v0-greedy_until b/lm-evaluation-harness/tests/testdata/drop-v0-greedy_until new file mode 100644 index 0000000000000000000000000000000000000000..6470b349d2e2a54c1ab113346885eb97c045a0ed --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/drop-v0-greedy_until @@ -0,0 +1 @@ +ca566c630d8ac853d5785d4b5c40a5137172c34b48af3350e1f79e6d548b36ba \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/drop-v0-res.json b/lm-evaluation-harness/tests/testdata/drop-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..9384ca72fe6c84f3a6a9c419b82a7dd7f39bf7d1 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/drop-v0-res.json @@ -0,0 +1 @@ +{"results": {"drop": {"em": 0.0, "em_stderr": 0.0, "f1": 0.0, "f1_stderr": 0.0}}, "versions": {"drop": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/drop-v1-greedy_until b/lm-evaluation-harness/tests/testdata/drop-v1-greedy_until new file mode 100644 index 0000000000000000000000000000000000000000..3b2b697c91962eb160da3950bb22e45889c265e6 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/drop-v1-greedy_until @@ -0,0 +1 @@ +a670f911ab2999d72db15f534b22703d19e7837edbda4f9f199ad587f7aae6b2 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/drop-v1-res.json b/lm-evaluation-harness/tests/testdata/drop-v1-res.json new file mode 100644 index 0000000000000000000000000000000000000000..8f397b410df7a77e25fd2916787b6050a5806d1d --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/drop-v1-res.json @@ -0,0 +1 @@ +{"results": {"drop": {"em": 0.0, "em_stderr": 0.0, "f1": 0.0, "f1_stderr": 0.0}}, "versions": {"drop": 1}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/ethics_cm-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/ethics_cm-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..69289144e0e3ceb0051596d5768b70667f7d19a8 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/ethics_cm-v0-loglikelihood @@ -0,0 +1 @@ +92d136ebb2bd86cd036e61699ad9a1417dbb48651f0a3afa5045cf57cef5a3f6 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/ethics_cm-v0-res.json b/lm-evaluation-harness/tests/testdata/ethics_cm-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..f81a700903262aec7eae2b4c39260f3f2c8f1dd0 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/ethics_cm-v0-res.json @@ -0,0 +1 @@ +{"results": {"ethics_cm": {"acc": 0.49987129987129986, "acc_stderr": 0.008022881531793336}}, "versions": {"ethics_cm": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/ethics_deontology-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/ethics_deontology-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..ab01349737c063432656f3951ae913a63a85adba --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/ethics_deontology-v0-loglikelihood @@ -0,0 +1 @@ +74ecebe322457d70afc16fde848978410a09b854dc65c47f428d100bd1593248 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/ethics_deontology-v0-res.json b/lm-evaluation-harness/tests/testdata/ethics_deontology-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..3af24f414a42803984877a710b95c037187984b9 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/ethics_deontology-v0-res.json @@ -0,0 +1 @@ +{"results": {"ethics_deontology": {"acc": 0.503615127919911, "acc_stderr": 0.008338908432085105, "em": 0.07119021134593993}}, "versions": {"ethics_deontology": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/ethics_justice-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/ethics_justice-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..cc18a7e67b6f38aaf759bb9073314da42b86f992 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/ethics_justice-v0-loglikelihood @@ -0,0 +1 @@ +d7dfc44fea507b5c5c3a8218f79ed8197da8599ebb396d85feb91c25512126b6 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/ethics_justice-v0-res.json b/lm-evaluation-harness/tests/testdata/ethics_justice-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..39efbc506acfd90b86362185d28d43090aeb7d1c --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/ethics_justice-v0-res.json @@ -0,0 +1 @@ +{"results": {"ethics_justice": {"acc": 0.49556213017751477, "acc_stderr": 0.009616784279885177, "em": 0.057692307692307696}}, "versions": {"ethics_justice": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/ethics_utilitarianism-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/ethics_utilitarianism-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..0c01f548806c747150690d942f7def8b2d98f2a2 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/ethics_utilitarianism-v0-loglikelihood @@ -0,0 +1 @@ +88872f1ed1b203f9649a4ced4fb4627d18c17af455d713de6e17c05eced4ec60 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/ethics_utilitarianism-v0-res.json b/lm-evaluation-harness/tests/testdata/ethics_utilitarianism-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..857af346b47d7ce11ee4192b928608a2111776f4 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/ethics_utilitarianism-v0-res.json @@ -0,0 +1 @@ +{"results": {"ethics_utilitarianism": {"acc": 0.49771214642262895, "acc_stderr": 0.007211546310787838}}, "versions": {"ethics_utilitarianism": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/ethics_utilitarianism_original-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/ethics_utilitarianism_original-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..bd3ff6c459c5a5739b233dd86c5434f64bbc1b16 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/ethics_utilitarianism_original-v0-loglikelihood @@ -0,0 +1 @@ +5b42ba1faf5ece6a6ec9a3976ce79c1fac8df5b98272aab85457188c2142693c \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/ethics_utilitarianism_original-v0-res.json b/lm-evaluation-harness/tests/testdata/ethics_utilitarianism_original-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..16940c8f5a7dd9ebb1d73298346ab1d19811ec90 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/ethics_utilitarianism_original-v0-res.json @@ -0,0 +1 @@ +{"results": {"ethics_utilitarianism_original": {"acc": 0.5214226289517471, "acc_stderr": 0.007204999520618661}}, "versions": {"ethics_utilitarianism_original": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/ethics_virtue-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/ethics_virtue-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..48652c4689e2be24972881d0abff497d203ace9a --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/ethics_virtue-v0-loglikelihood @@ -0,0 +1 @@ +8021db8de46850090ddae6e6ec2d382029c3027b7c69884607503f916d09b709 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/ethics_virtue-v0-res.json b/lm-evaluation-harness/tests/testdata/ethics_virtue-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..cf3e02d82662bc1c4de5f1cf3dd9442b321de623 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/ethics_virtue-v0-res.json @@ -0,0 +1 @@ +{"results": {"ethics_virtue": {"acc": 0.5035175879396985, "acc_stderr": 0.0070893491553555765, "em": 0.036180904522613064}}, "versions": {"ethics_virtue": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/gsm8k-v0-greedy_until b/lm-evaluation-harness/tests/testdata/gsm8k-v0-greedy_until new file mode 100644 index 0000000000000000000000000000000000000000..d49400007f95ecd048628bb2f1cadf92132bef24 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/gsm8k-v0-greedy_until @@ -0,0 +1 @@ +e7292dbdd7fd8419ba954f2e0701e04c8d0e8842fe053dbf2fe47d926630e35e \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/gsm8k-v0-res.json b/lm-evaluation-harness/tests/testdata/gsm8k-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..fb6514a0e750d4e3737cf33766fcc851f79bfa48 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/gsm8k-v0-res.json @@ -0,0 +1 @@ +{"results": {"gsm8k": {"acc": 0.0, "acc_stderr": 0.0}}, "versions": {"gsm8k": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/headqa-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/headqa-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..9129d834b6037cda3db655064d6c18bb3dccfb54 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/headqa-v0-loglikelihood @@ -0,0 +1 @@ +767ca34d9714edd9fb030ddbcc35a64e5180d1e247b0cb557fbb22fdf971ad1f \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/headqa-v0-res.json b/lm-evaluation-harness/tests/testdata/headqa-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..adc093cf62c2f807a0f413d0ecc200879931a5b7 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/headqa-v0-res.json @@ -0,0 +1 @@ +{"results": {"headqa": {"acc": 0.23559445660102116, "acc_norm": 0.25018234865062, "acc_norm_stderr": 0.008272783230806014, "acc_stderr": 0.008105688874297972}}, "versions": {"headqa": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/headqa_en-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/headqa_en-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..11f07878fb5452ac334eaf0daf276aa8684124f6 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/headqa_en-v0-loglikelihood @@ -0,0 +1 @@ +09da45119b12a0144e3081f8fb790c2a22af7b9c3aac42f54423d348a711fbf5 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/headqa_en-v0-res.json b/lm-evaluation-harness/tests/testdata/headqa_en-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..6ac5a9c0b8e70a47f2c985713a50336c68b11382 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/headqa_en-v0-res.json @@ -0,0 +1 @@ +{"results": {"headqa_en": {"acc": 0.23559445660102116, "acc_norm": 0.2447118891320204, "acc_norm_stderr": 0.008211629406841468, "acc_stderr": 0.008105688874297972}}, "versions": {"headqa_en": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/headqa_es-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/headqa_es-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..9129d834b6037cda3db655064d6c18bb3dccfb54 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/headqa_es-v0-loglikelihood @@ -0,0 +1 @@ +767ca34d9714edd9fb030ddbcc35a64e5180d1e247b0cb557fbb22fdf971ad1f \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/headqa_es-v0-res.json b/lm-evaluation-harness/tests/testdata/headqa_es-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..0964db9bbb8a6b0ca129c3e069151f334558de54 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/headqa_es-v0-res.json @@ -0,0 +1 @@ +{"results": {"headqa_es": {"acc": 0.23559445660102116, "acc_norm": 0.25018234865062, "acc_norm_stderr": 0.008272783230806014, "acc_stderr": 0.008105688874297972}}, "versions": {"headqa_es": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hellaswag-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hellaswag-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..c679a3e311759f4a00707b7454e0e8be4bcdfff0 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hellaswag-v0-loglikelihood @@ -0,0 +1 @@ +abb808c97d6529eda6c11067837a132c62d25cba0394d720f80cca6df9f7196e \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hellaswag-v0-res.json b/lm-evaluation-harness/tests/testdata/hellaswag-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..6be94a640950b2451775fddccbf80060c4a673b0 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hellaswag-v0-res.json @@ -0,0 +1 @@ +{"results": {"hellaswag": {"acc": 0.24965146385182235, "acc_norm": 0.24756024696275641, "acc_norm_stderr": 0.004307128573285236, "acc_stderr": 0.004319267432460666}}, "versions": {"hellaswag": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-abstract_algebra-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-abstract_algebra-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..d0d0fe872b24d1304c932ffde5546a70b125e100 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-abstract_algebra-v0-loglikelihood @@ -0,0 +1 @@ +e35d1eeb356ac1084d4e9773f028cb3c81ba1c6e5574d598ac4a78aa467cd797 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-abstract_algebra-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-abstract_algebra-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..dc2c9a0d7d4d4a18ee7c8cb0e266a29fa5bd48f6 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-abstract_algebra-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-abstract_algebra": {"acc": 0.32, "acc_norm": 0.34, "acc_norm_stderr": 0.04760952285695235, "acc_stderr": 0.04688261722621504}}, "versions": {"hendrycksTest-abstract_algebra": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-anatomy-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-anatomy-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..a7ae5fa705e58cf0e7c06ca0fe84a186d24b506f --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-anatomy-v0-loglikelihood @@ -0,0 +1 @@ +bf05e04ed8cf61cf3aad294ed3f5a16137775ffdd20f1b129022ddffc1251768 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-anatomy-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-anatomy-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..67bc2e7be6de4ba9d6b9aa40c0d45cd60d7d506b --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-anatomy-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-anatomy": {"acc": 0.2222222222222222, "acc_norm": 0.23703703703703705, "acc_norm_stderr": 0.03673731683969506, "acc_stderr": 0.0359144408419697}}, "versions": {"hendrycksTest-anatomy": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-astronomy-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-astronomy-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..8ecb637cfe4eaf6d3bbca863c7bab6188b85425b --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-astronomy-v0-loglikelihood @@ -0,0 +1 @@ +bed1e47127cc2893c6aef63b9a0909cca31aa351a703da2a166b01cae03c3311 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-astronomy-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-astronomy-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..d3626ccf80f233702478886fffeede1f587ad2fb --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-astronomy-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-astronomy": {"acc": 0.2565789473684211, "acc_norm": 0.29605263157894735, "acc_norm_stderr": 0.03715062154998904, "acc_stderr": 0.0355418036802569}}, "versions": {"hendrycksTest-astronomy": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-business_ethics-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-business_ethics-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..a0f8b7c09b3b6307123f1328c51c1dcfb797aed2 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-business_ethics-v0-loglikelihood @@ -0,0 +1 @@ +b3b27e9dbad587377d3c8cab1072782de883e245da93a563bd8b3099017b1fc0 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-business_ethics-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-business_ethics-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..dcc5116204283941b74dfea97e3a1ce5edd9dc27 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-business_ethics-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-business_ethics": {"acc": 0.29, "acc_norm": 0.27, "acc_norm_stderr": 0.044619604333847394, "acc_stderr": 0.045604802157206845}}, "versions": {"hendrycksTest-business_ethics": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-clinical_knowledge-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-clinical_knowledge-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..86f54245d557e0091d1166b7ffb2029520e566e9 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-clinical_knowledge-v0-loglikelihood @@ -0,0 +1 @@ +fbcb7ce507e0675d811e71e10a67c8d05a6605e29036f46776e04a6588cefbda \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-clinical_knowledge-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-clinical_knowledge-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..596bb28a93f52c857b6a39d416114c12c7ea9147 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-clinical_knowledge-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-clinical_knowledge": {"acc": 0.23773584905660378, "acc_norm": 0.27169811320754716, "acc_norm_stderr": 0.027377706624670713, "acc_stderr": 0.02619980880756191}}, "versions": {"hendrycksTest-clinical_knowledge": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-college_biology-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-college_biology-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..7f665ef4a1bd06ecfd30d999ae6880c00ba849cf --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-college_biology-v0-loglikelihood @@ -0,0 +1 @@ +c29e4e67ff91af29b9434884874414d1b1b32ccc32903c6b1639469b19907419 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-college_biology-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-college_biology-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..6705b9cad27c7f1eb647b513861646faaccad584 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-college_biology-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-college_biology": {"acc": 0.24305555555555555, "acc_norm": 0.2361111111111111, "acc_norm_stderr": 0.03551446610810826, "acc_stderr": 0.03586879280080341}}, "versions": {"hendrycksTest-college_biology": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-college_chemistry-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-college_chemistry-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..52a255e82a35b8d084459e72140f30f26ef8c57f --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-college_chemistry-v0-loglikelihood @@ -0,0 +1 @@ +044752b21540db95118b8cbe7e75c4c9b8758e27df56543deaeadec7f749a28d \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-college_chemistry-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-college_chemistry-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..4dc95a151ac2da73b3c5eb23e3fe24a7ccc8024d --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-college_chemistry-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-college_chemistry": {"acc": 0.28, "acc_norm": 0.26, "acc_norm_stderr": 0.04408440022768078, "acc_stderr": 0.04512608598542127}}, "versions": {"hendrycksTest-college_chemistry": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-college_computer_science-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-college_computer_science-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..695bc8c31592a4c33d70d5d07a8c5b523d9bd3cc --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-college_computer_science-v0-loglikelihood @@ -0,0 +1 @@ +4ea26ad780290429ac5a3317559c154848d662bd40532c966458ba6f2a32d0a3 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-college_computer_science-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-college_computer_science-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..aea595c09f5baf6d21867c47fd5e42152244f555 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-college_computer_science-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-college_computer_science": {"acc": 0.22, "acc_norm": 0.24, "acc_norm_stderr": 0.04292346959909282, "acc_stderr": 0.041633319989322695}}, "versions": {"hendrycksTest-college_computer_science": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-college_mathematics-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-college_mathematics-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..a840b6b6420053c343787f08d8d723ab5ba5c1d3 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-college_mathematics-v0-loglikelihood @@ -0,0 +1 @@ +e9fe80752686527281f834d2397875b4580581434b94799f9de6aaa450bd73ff \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-college_mathematics-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-college_mathematics-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..766b3388ed88d61e2c17ed2a35110879160c5f7f --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-college_mathematics-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-college_mathematics": {"acc": 0.18, "acc_norm": 0.2, "acc_norm_stderr": 0.04020151261036844, "acc_stderr": 0.038612291966536955}}, "versions": {"hendrycksTest-college_mathematics": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-college_medicine-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-college_medicine-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..2fb96497d12f9b72dbbd38f0d64aa75615bfe14b --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-college_medicine-v0-loglikelihood @@ -0,0 +1 @@ +dd6e0a9be1407890e9f8cd4434fb6aa4752ab3d2473837fd465ad99f60ad685e \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-college_medicine-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-college_medicine-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..524552c9bb99335a9a7bee73076bc633b7eb10e3 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-college_medicine-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-college_medicine": {"acc": 0.27167630057803466, "acc_norm": 0.2543352601156069, "acc_norm_stderr": 0.0332055644308557, "acc_stderr": 0.03391750322321659}}, "versions": {"hendrycksTest-college_medicine": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-college_physics-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-college_physics-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..7c2e2f4bf73266d532c7514c98defcba0133f231 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-college_physics-v0-loglikelihood @@ -0,0 +1 @@ +704a7671ef981fb95594782bc446dd632e87ebdbe89436a0603b714fb5786c75 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-college_physics-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-college_physics-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..97e56f2ae62e6b0012d49c6a7a55614a6d6eaf58 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-college_physics-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-college_physics": {"acc": 0.23529411764705882, "acc_norm": 0.23529411764705882, "acc_norm_stderr": 0.04220773659171453, "acc_stderr": 0.04220773659171452}}, "versions": {"hendrycksTest-college_physics": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-computer_security-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-computer_security-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..d4c0ee2d78364c0275d984a4ef43cfcedbaf55ed --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-computer_security-v0-loglikelihood @@ -0,0 +1 @@ +a8a1892d1906cc3e7ffd321043f0a60f3b8b69ef76e5c6ff03c6ea41dc87d0cb \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-computer_security-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-computer_security-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..60f02eba9cb04602d8b67d67269d8b82e0930721 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-computer_security-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-computer_security": {"acc": 0.24, "acc_norm": 0.27, "acc_norm_stderr": 0.044619604333847394, "acc_stderr": 0.042923469599092816}}, "versions": {"hendrycksTest-computer_security": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-conceptual_physics-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-conceptual_physics-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..05c4db0e2290998cb650c11373f0947c3be8f297 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-conceptual_physics-v0-loglikelihood @@ -0,0 +1 @@ +622f191ccfc7a597d99f39897ebe3f95a9ddce0e662fcfb411aa554b289bb355 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-conceptual_physics-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-conceptual_physics-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..1388bcdcd9a7283decbf3680283a2cc4cfc7cfde --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-conceptual_physics-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-conceptual_physics": {"acc": 0.2680851063829787, "acc_norm": 0.2553191489361702, "acc_norm_stderr": 0.028504856470514185, "acc_stderr": 0.028957342788342347}}, "versions": {"hendrycksTest-conceptual_physics": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-econometrics-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-econometrics-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..ed3332eddaf041c82908352c43cf8d9187b8f381 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-econometrics-v0-loglikelihood @@ -0,0 +1 @@ +cde76ba2c7382b4876e17136c94f52aca2774e50342ab757b2a2d18da370dcb6 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-econometrics-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-econometrics-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..4656fac3c3026ec7e137ce8f49e4796fefe5e24f --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-econometrics-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-econometrics": {"acc": 0.24561403508771928, "acc_norm": 0.24561403508771928, "acc_norm_stderr": 0.04049339297748142, "acc_stderr": 0.040493392977481425}}, "versions": {"hendrycksTest-econometrics": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-electrical_engineering-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-electrical_engineering-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..9c9e72efdf98ed9afb4881647929246433e1f857 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-electrical_engineering-v0-loglikelihood @@ -0,0 +1 @@ +b9b5d8b8bb02696302ec6bc2a99bf987a5504d3bae0e529d2c8f263538c97518 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-electrical_engineering-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-electrical_engineering-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..13b76c1d5f94218128b2038d55bd300faf66ff44 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-electrical_engineering-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-electrical_engineering": {"acc": 0.2689655172413793, "acc_norm": 0.2827586206896552, "acc_norm_stderr": 0.037528339580033376, "acc_stderr": 0.036951833116502325}}, "versions": {"hendrycksTest-electrical_engineering": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-elementary_mathematics-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-elementary_mathematics-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..e281f72feb428451f27dbaba80408c468ef51bce --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-elementary_mathematics-v0-loglikelihood @@ -0,0 +1 @@ +6b21f5cd5606268421a667152ec989424b66905c02adbab8d4ff6bb9d21b77d1 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-elementary_mathematics-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-elementary_mathematics-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..84cd983ee9d33f831ee397ffd8b11990b70a4b60 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-elementary_mathematics-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-elementary_mathematics": {"acc": 0.2724867724867725, "acc_norm": 0.2830687830687831, "acc_norm_stderr": 0.023201392938194978, "acc_stderr": 0.022930973071633345}}, "versions": {"hendrycksTest-elementary_mathematics": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-formal_logic-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-formal_logic-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..ef6bec3f70adb9b8df43583cf76e6cd865831b0b --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-formal_logic-v0-loglikelihood @@ -0,0 +1 @@ +c0d0f0c008a5f3faf2f6f4268d87bbc09c40bb66ae08cf38eea0bf2e519c5a59 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-formal_logic-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-formal_logic-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..acde01d4d7d45333322eaa4a07edf42ec414d08c --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-formal_logic-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-formal_logic": {"acc": 0.25396825396825395, "acc_norm": 0.2698412698412698, "acc_norm_stderr": 0.03970158273235172, "acc_stderr": 0.03893259610604674}}, "versions": {"hendrycksTest-formal_logic": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-global_facts-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-global_facts-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..a4751fdbfad7b614f9ec059a78130426e1d8a39c --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-global_facts-v0-loglikelihood @@ -0,0 +1 @@ +9fdc85240b8170839278b1e883ee0868611d84dce202cb8aa037c841ec76d089 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-global_facts-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-global_facts-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..d2fff47bcbaaaead17eceef0ca09cd45014c5aac --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-global_facts-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-global_facts": {"acc": 0.23, "acc_norm": 0.23, "acc_norm_stderr": 0.04229525846816507, "acc_stderr": 0.04229525846816507}}, "versions": {"hendrycksTest-global_facts": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_biology-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_biology-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..1e2c01e2b19082144373a13ee25e3e68bf8df588 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_biology-v0-loglikelihood @@ -0,0 +1 @@ +d4dc051f37a49dc75c218741e87bc826fd44f31ee1309b55e0f33bd191c1bc78 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_biology-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_biology-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..a666d9ce9c969f808ea84909730cee046ccc6294 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_biology-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-high_school_biology": {"acc": 0.23870967741935484, "acc_norm": 0.2709677419354839, "acc_norm_stderr": 0.025284416114900152, "acc_stderr": 0.024251071262208834}}, "versions": {"hendrycksTest-high_school_biology": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_chemistry-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_chemistry-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..d0ca97d6a58d8dae225d36636ef21b0fd1e50fdf --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_chemistry-v0-loglikelihood @@ -0,0 +1 @@ +f4f338e45415c4b5ee7f1d249155bcd910c8401bd1436760a5ec61cb6bb211b6 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_chemistry-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_chemistry-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..2d81594963cefe41f139a813fcdc16c0f247f9ed --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_chemistry-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-high_school_chemistry": {"acc": 0.2857142857142857, "acc_norm": 0.2660098522167488, "acc_norm_stderr": 0.031089826002937523, "acc_stderr": 0.031785297106427496}}, "versions": {"hendrycksTest-high_school_chemistry": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_computer_science-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_computer_science-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..a421564657975a25dedfd1c8cf38ef0e0ea4df9c --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_computer_science-v0-loglikelihood @@ -0,0 +1 @@ +870d5a6300c527077aaf6baa3e750e75fa840b41657cf82549f39b768b14862d \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_computer_science-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_computer_science-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..bbc2dacf5f5ac0b14327f0637b4b1aabea7a6167 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_computer_science-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-high_school_computer_science": {"acc": 0.2, "acc_norm": 0.22, "acc_norm_stderr": 0.04163331998932269, "acc_stderr": 0.04020151261036845}}, "versions": {"hendrycksTest-high_school_computer_science": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_european_history-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_european_history-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..eec5858ef9a22ba66ee0627646b5ce98f2b0326d --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_european_history-v0-loglikelihood @@ -0,0 +1 @@ +d8070e113be9d420fef5578cb69c70df4ea5118f9b18553023fd9efd5ff0b7f4 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_european_history-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_european_history-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..b5cea9cbe310db37d488984f3ff6aa57921576d9 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_european_history-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-high_school_european_history": {"acc": 0.23636363636363636, "acc_norm": 0.24242424242424243, "acc_norm_stderr": 0.03346409881055953, "acc_stderr": 0.033175059300091805}}, "versions": {"hendrycksTest-high_school_european_history": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_geography-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_geography-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..ac80d178809ddbacc3aeb8ff368d8a68605a6430 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_geography-v0-loglikelihood @@ -0,0 +1 @@ +add45970ea3865be7c7a31f788a835949f6937ac73f699b122ca56a3431e95f8 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_geography-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_geography-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..0fb76aa9ba10e63a931ec3707ac6d9f3d84292aa --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_geography-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-high_school_geography": {"acc": 0.2474747474747475, "acc_norm": 0.2777777777777778, "acc_norm_stderr": 0.03191178226713547, "acc_stderr": 0.03074630074212452}}, "versions": {"hendrycksTest-high_school_geography": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_government_and_politics-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_government_and_politics-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..12ea726b4ba81f264017a7fd71d18a6ac318b0ab --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_government_and_politics-v0-loglikelihood @@ -0,0 +1 @@ +11f40d8f48ba5cd739e21d54c3c04d3761f81df5cb7ddd77df868d24ced44b49 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_government_and_politics-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_government_and_politics-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..16cc02ff0a897dda3a6c6dc97e9b7815ea120fc2 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_government_and_politics-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-high_school_government_and_politics": {"acc": 0.24352331606217617, "acc_norm": 0.23834196891191708, "acc_norm_stderr": 0.03074890536390988, "acc_stderr": 0.030975436386845436}}, "versions": {"hendrycksTest-high_school_government_and_politics": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_macroeconomics-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_macroeconomics-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..c0106d373dcf6136b147bb3787fed6c9c8a3da8f --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_macroeconomics-v0-loglikelihood @@ -0,0 +1 @@ +ce4faae2fb6628caa48f6fc74cbc848880db49e6ff51079392778a2322bcefef \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_macroeconomics-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_macroeconomics-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..fb6835039c9d68b5cf5d52244a349c1b8a964c5c --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_macroeconomics-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-high_school_macroeconomics": {"acc": 0.2230769230769231, "acc_norm": 0.22564102564102564, "acc_norm_stderr": 0.021193632525148522, "acc_stderr": 0.021107730127244}}, "versions": {"hendrycksTest-high_school_macroeconomics": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_mathematics-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_mathematics-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..dc86769fa93781e03ca8f7e7b3493b39338bcdaa --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_mathematics-v0-loglikelihood @@ -0,0 +1 @@ +ab368d16fc4648ad27940f71abd266366663f51db612f732a0b9b0eea28de9f8 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_mathematics-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_mathematics-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..cb3a3ec0688cdc4905ffad6e17c91d59c9330572 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_mathematics-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-high_school_mathematics": {"acc": 0.22592592592592592, "acc_norm": 0.24814814814814815, "acc_norm_stderr": 0.0263357394040558, "acc_stderr": 0.025497532639609553}}, "versions": {"hendrycksTest-high_school_mathematics": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_microeconomics-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_microeconomics-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..37962bf9fb93bc8f49fa83af34c30ac0ef49df09 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_microeconomics-v0-loglikelihood @@ -0,0 +1 @@ +513b998585ebc1ebdefca6435b7c84fd73dc36fc80321a22503467f04efed23e \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_microeconomics-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_microeconomics-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..cf698d181c95ef88e774204df6f92622116d690c --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_microeconomics-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-high_school_microeconomics": {"acc": 0.24369747899159663, "acc_norm": 0.22268907563025211, "acc_norm_stderr": 0.027025433498882378, "acc_stderr": 0.027886828078380558}}, "versions": {"hendrycksTest-high_school_microeconomics": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_physics-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_physics-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..49a780bc97953db32716ccc580390c5d21cfc252 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_physics-v0-loglikelihood @@ -0,0 +1 @@ +dae59e82d3d4d8dec82239d9620b72cc47bb6efbe2f1c2f9b9d23e849c9c5e32 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_physics-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_physics-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..b6b3bb9d012756280cf8a0ba68d4011fe9089e39 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_physics-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-high_school_physics": {"acc": 0.2582781456953642, "acc_norm": 0.271523178807947, "acc_norm_stderr": 0.03631329803969653, "acc_stderr": 0.035737053147634576}}, "versions": {"hendrycksTest-high_school_physics": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_psychology-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_psychology-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..0f39ddfde7066ac8c577156336644c35a543afbb --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_psychology-v0-loglikelihood @@ -0,0 +1 @@ +0e4c8d13806d3696167e40544d2d114c557c10c74bc61fcb9c51bbfced0266ef \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_psychology-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_psychology-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..42b781149bff323130b4491463168f03bdfbb9a9 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_psychology-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-high_school_psychology": {"acc": 0.24587155963302754, "acc_norm": 0.23302752293577983, "acc_norm_stderr": 0.018125669180861493, "acc_stderr": 0.018461940968708436}}, "versions": {"hendrycksTest-high_school_psychology": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_statistics-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_statistics-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..8a915ef7fc0ab9a7c290867450265a7cadd40494 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_statistics-v0-loglikelihood @@ -0,0 +1 @@ +33d1d6eaaa2c3a944bf49d3f220a4efc328d7c3b3465b7cec40ae36d8984b75f \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_statistics-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_statistics-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..4c6a21d7dac4cd7b6fa217e8bebf34d959554a7a --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_statistics-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-high_school_statistics": {"acc": 0.2962962962962963, "acc_norm": 0.3055555555555556, "acc_norm_stderr": 0.03141554629402544, "acc_stderr": 0.03114144782353604}}, "versions": {"hendrycksTest-high_school_statistics": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_us_history-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_us_history-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..e05b91503e0a2c2c8bb8ef34af16e87c902c31f9 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_us_history-v0-loglikelihood @@ -0,0 +1 @@ +8c65c1a28330dd001d395ac11f1bb80c3b33f5935f503e74067aef6e9e1d9d9b \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_us_history-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_us_history-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..5b7a76909c08ab9c47c5e1eb6c06945e990fb639 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_us_history-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-high_school_us_history": {"acc": 0.29901960784313725, "acc_norm": 0.28431372549019607, "acc_norm_stderr": 0.03166009679399814, "acc_stderr": 0.03213325717373618}}, "versions": {"hendrycksTest-high_school_us_history": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_world_history-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_world_history-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..228dfe072cd02f94bced495f271c5cc108850719 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_world_history-v0-loglikelihood @@ -0,0 +1 @@ +1c8b994bd9a63ec874fc8d0e3a27077118b7adc472306b2fd6c55635a78b9d52 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_world_history-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_world_history-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..ca1bf95b9d2d37c2b9cbe75efd7f1e3fd88ecdcf --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-high_school_world_history-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-high_school_world_history": {"acc": 0.23628691983122363, "acc_norm": 0.24472573839662448, "acc_norm_stderr": 0.02798569938703642, "acc_stderr": 0.027652153144159263}}, "versions": {"hendrycksTest-high_school_world_history": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-human_aging-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-human_aging-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..d34fa529800590ecc8e199fdb9d141c99b8c6876 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-human_aging-v0-loglikelihood @@ -0,0 +1 @@ +0880b3a78f8d7b17ffc612031427b9085367cf65dabe2a68c4b64e3171d17e88 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-human_aging-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-human_aging-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..061678f2e4f30402e1c44da7c4a23cae0e57bedf --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-human_aging-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-human_aging": {"acc": 0.21524663677130046, "acc_norm": 0.17937219730941703, "acc_norm_stderr": 0.025749819569192804, "acc_stderr": 0.02758406660220827}}, "versions": {"hendrycksTest-human_aging": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-human_sexuality-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-human_sexuality-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..b3d3ae438c1fc59930d1d4ba053d73c38b6d9c07 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-human_sexuality-v0-loglikelihood @@ -0,0 +1 @@ +4b07922fa1d549b655c21440b13d869263ce7dd9771d8147c450f11c91d26c10 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-human_sexuality-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-human_sexuality-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..091d7352ce1b260f6acbd1338b7d54c5716d23ce --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-human_sexuality-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-human_sexuality": {"acc": 0.22137404580152673, "acc_norm": 0.22900763358778625, "acc_norm_stderr": 0.036853466317118506, "acc_stderr": 0.0364129708131373}}, "versions": {"hendrycksTest-human_sexuality": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-international_law-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-international_law-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..2b6aa8d605765b06a262877dec34cd156d0a66f9 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-international_law-v0-loglikelihood @@ -0,0 +1 @@ +ea9b2cefd27959db564168f6ad1169a5eaa012fc5a5d5b8faf9e34d94e335dc1 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-international_law-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-international_law-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..bd4edd2394a4ccaf2d75c578a9f45ad614657dd8 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-international_law-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-international_law": {"acc": 0.2396694214876033, "acc_norm": 0.3140495867768595, "acc_norm_stderr": 0.042369647530410164, "acc_stderr": 0.03896878985070417}}, "versions": {"hendrycksTest-international_law": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-jurisprudence-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-jurisprudence-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..3d55d21e0294d78ebb728920d0651ccf6f9150b7 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-jurisprudence-v0-loglikelihood @@ -0,0 +1 @@ +cac440189f1ec778e82f4975d88b74689553ecc5116aaa7f76587a50c1a610e0 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-jurisprudence-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-jurisprudence-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..4ef181974956e3f899121ac46dc5e192231d1a65 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-jurisprudence-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-jurisprudence": {"acc": 0.25, "acc_norm": 0.3148148148148148, "acc_norm_stderr": 0.04489931073591312, "acc_stderr": 0.04186091791394607}}, "versions": {"hendrycksTest-jurisprudence": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-logical_fallacies-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-logical_fallacies-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..a5807b5831206a8cb29b2e99c02c7c6025dd6f25 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-logical_fallacies-v0-loglikelihood @@ -0,0 +1 @@ +2e9449dd803f9e2334dc562d9f04031fd013ed36b883b44ab500533a5dbbface \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-logical_fallacies-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-logical_fallacies-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..c5cf5cb467d80051cea569ab30ccc20d697e1e57 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-logical_fallacies-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-logical_fallacies": {"acc": 0.20245398773006135, "acc_norm": 0.2147239263803681, "acc_norm_stderr": 0.03226219377286774, "acc_stderr": 0.03157065078911902}}, "versions": {"hendrycksTest-logical_fallacies": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-machine_learning-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-machine_learning-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..53e498ddd480dfaf3994eba4069ead8a28694784 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-machine_learning-v0-loglikelihood @@ -0,0 +1 @@ +7a7138821a66ef946e427b40344cf7f1a916a2926995a85ef731a3bee40cb7ce \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-machine_learning-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-machine_learning-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..26be724f2426d0a7b204b2f4dee509597e85ab41 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-machine_learning-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-machine_learning": {"acc": 0.23214285714285715, "acc_norm": 0.22321428571428573, "acc_norm_stderr": 0.039523019677025116, "acc_stderr": 0.04007341809755806}}, "versions": {"hendrycksTest-machine_learning": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-management-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-management-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..571873985758661a0feed74e164f606318e58d8c --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-management-v0-loglikelihood @@ -0,0 +1 @@ +355489f4bd176ab84db5ef4c03d56ddeeeb1b0ad69827122b2d800e1cdc7e5f0 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-management-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-management-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..7a84623fabf793b7748d34c18f4c358649f31a97 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-management-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-management": {"acc": 0.24271844660194175, "acc_norm": 0.2621359223300971, "acc_norm_stderr": 0.043546310772605956, "acc_stderr": 0.04245022486384495}}, "versions": {"hendrycksTest-management": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-marketing-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-marketing-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..1d241a97733c081a3f00280cfbedc411c0570001 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-marketing-v0-loglikelihood @@ -0,0 +1 @@ +b4fa0681fe54671a80509779d4338d744097a7206687f62977df7145dfa74a66 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-marketing-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-marketing-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..2cc7a93f1c3c2b4747d4ce739ffbcd522fc50224 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-marketing-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-marketing": {"acc": 0.2863247863247863, "acc_norm": 0.2905982905982906, "acc_norm_stderr": 0.029745048572674043, "acc_stderr": 0.029614323690456648}}, "versions": {"hendrycksTest-marketing": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-medical_genetics-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-medical_genetics-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..48d49de8399fba6cfb50dd98d3cbcf8d39388ab2 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-medical_genetics-v0-loglikelihood @@ -0,0 +1 @@ +db6141246889a19dd3f6b9109f314d49c1a70f7a98795858804378b095c4a2fe \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-medical_genetics-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-medical_genetics-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..eac53bcf4a7610934b697d6d19f53ecdf5d4a4ad --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-medical_genetics-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-medical_genetics": {"acc": 0.27, "acc_norm": 0.29, "acc_norm_stderr": 0.04560480215720684, "acc_stderr": 0.0446196043338474}}, "versions": {"hendrycksTest-medical_genetics": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-miscellaneous-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-miscellaneous-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..b09e99721b8ec71dc85c7ed0798d55a6e0274860 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-miscellaneous-v0-loglikelihood @@ -0,0 +1 @@ +972dd88dbbaf09d14766e243cfc233425e7c01a26dbc61bdb9eeefa788822331 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-miscellaneous-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-miscellaneous-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..5c7859eb3a80a849deee7d67d37f71a84c8eeaf6 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-miscellaneous-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-miscellaneous": {"acc": 0.23499361430395913, "acc_norm": 0.2515964240102171, "acc_norm_stderr": 0.015517322365529622, "acc_stderr": 0.015162024152278445}}, "versions": {"hendrycksTest-miscellaneous": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-moral_disputes-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-moral_disputes-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..953fc3be48759378aea33eb767cb7367514a5de9 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-moral_disputes-v0-loglikelihood @@ -0,0 +1 @@ +d6ef028022c02b69d1516973e08bebaa14d8debcf2589a2bb124823178202d20 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-moral_disputes-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-moral_disputes-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..26ea1c2a75ccfb96af880ee30eef11520e9ea39c --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-moral_disputes-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-moral_disputes": {"acc": 0.24855491329479767, "acc_norm": 0.27167630057803466, "acc_norm_stderr": 0.023948512905468365, "acc_stderr": 0.023267528432100174}}, "versions": {"hendrycksTest-moral_disputes": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-moral_scenarios-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-moral_scenarios-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..d5ea0d8156ae4efaa0f7568ae8fd3a8ed3992d37 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-moral_scenarios-v0-loglikelihood @@ -0,0 +1 @@ +a8e1882e77728b53c8b86312254d08320d8363fb606d746a8dd145b812f62cf5 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-moral_scenarios-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-moral_scenarios-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..62ec15971237e04f6c883c7369bbb50888494830 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-moral_scenarios-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-moral_scenarios": {"acc": 0.2547486033519553, "acc_norm": 0.25251396648044694, "acc_norm_stderr": 0.014530330201468654, "acc_stderr": 0.014572650383409158}}, "versions": {"hendrycksTest-moral_scenarios": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-nutrition-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-nutrition-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..2716bebe69e1c3884ba2e88056c87c5a5268b53e --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-nutrition-v0-loglikelihood @@ -0,0 +1 @@ +19e49d218f55ed5ec4bd1a6cd3f3388c6f620b81484e7abe8b298e5481c3044d \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-nutrition-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-nutrition-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..e2838f880581f7cf743d83ba99a26827c18a09de --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-nutrition-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-nutrition": {"acc": 0.24509803921568626, "acc_norm": 0.28104575163398693, "acc_norm_stderr": 0.025738854797818723, "acc_stderr": 0.02463004897982476}}, "versions": {"hendrycksTest-nutrition": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-philosophy-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-philosophy-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..3ea8ef0a0e3ddf5cc42c6305e1885e163399f38c --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-philosophy-v0-loglikelihood @@ -0,0 +1 @@ +a419204da36c2b7a70fa8909a3a804260cc3283c7e07917534dfb76216c77f46 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-philosophy-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-philosophy-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..ec9c1e79c117c88246fa596ca90821025c9786af --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-philosophy-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-philosophy": {"acc": 0.26366559485530544, "acc_norm": 0.2733118971061093, "acc_norm_stderr": 0.02531176597542612, "acc_stderr": 0.02502553850053234}}, "versions": {"hendrycksTest-philosophy": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-prehistory-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-prehistory-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..4c01847ef594713fee284436be6fe8d20d602554 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-prehistory-v0-loglikelihood @@ -0,0 +1 @@ +6983c560a562749f4f702249a3a6ae51fa495acc0643a980bf2cf52c6c5d4b95 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-prehistory-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-prehistory-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..e0163dd555e6c510aa24c5da5ad187ef52ed7c4d --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-prehistory-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-prehistory": {"acc": 0.2623456790123457, "acc_norm": 0.26851851851851855, "acc_norm_stderr": 0.024659685185967277, "acc_stderr": 0.02447722285613511}}, "versions": {"hendrycksTest-prehistory": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-professional_accounting-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-professional_accounting-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..fe5997427ef5df8be6d52709189b7baa8a410df9 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-professional_accounting-v0-loglikelihood @@ -0,0 +1 @@ +847418f7b22cd9b499e95fd73c40a2fbc40076895280cc2c560199c0c4c4f433 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-professional_accounting-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-professional_accounting-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..b665d57e234aa5b9f67f85da689bba952f930914 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-professional_accounting-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-professional_accounting": {"acc": 0.2553191489361702, "acc_norm": 0.26595744680851063, "acc_norm_stderr": 0.026358065698880582, "acc_stderr": 0.026011992930902006}}, "versions": {"hendrycksTest-professional_accounting": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-professional_law-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-professional_law-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..23fbfcf78e79595a64037311668042a1ec7f637f --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-professional_law-v0-loglikelihood @@ -0,0 +1 @@ +c38c9d5d84eeb7a5f3c4a34d6e70d7e15847b3c38f26e4b119c982bb935e118f \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-professional_law-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-professional_law-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..f15a9b34ff26e1382d04b4d6e41fdae6085b30c8 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-professional_law-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-professional_law": {"acc": 0.2561929595827901, "acc_norm": 0.2470664928292047, "acc_norm_stderr": 0.011015752255279352, "acc_stderr": 0.011149173153110582}}, "versions": {"hendrycksTest-professional_law": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-professional_medicine-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-professional_medicine-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..cc3c3be8c6c09ffccdf7dbfd318ea3928c87a769 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-professional_medicine-v0-loglikelihood @@ -0,0 +1 @@ +7a30599858398169cde61430c18efdd7fb4dcd09c34aa9baba70f0f8cf17a9f1 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-professional_medicine-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-professional_medicine-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..801ea2d224b7f4699c3a3defd7cde023e777a29e --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-professional_medicine-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-professional_medicine": {"acc": 0.23161764705882354, "acc_norm": 0.2536764705882353, "acc_norm_stderr": 0.02643132987078953, "acc_stderr": 0.025626533803777562}}, "versions": {"hendrycksTest-professional_medicine": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-professional_psychology-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-professional_psychology-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..9865854da311057c18f8a2571eedac2d02608df5 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-professional_psychology-v0-loglikelihood @@ -0,0 +1 @@ +92a5fad6e9ec700f84946faeccd399dda3569fb71837c9fb0c5c87f5ec29c43e \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-professional_psychology-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-professional_psychology-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..c6b33f4be16f9bc1ed04502ed0f1c121c3a9d1be --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-professional_psychology-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-professional_psychology": {"acc": 0.27124183006535946, "acc_norm": 0.2826797385620915, "acc_norm_stderr": 0.01821726955205344, "acc_stderr": 0.01798661530403031}}, "versions": {"hendrycksTest-professional_psychology": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-public_relations-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-public_relations-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..8f7b30ba8823a0a0d8fc94f69ef64d362835e0db --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-public_relations-v0-loglikelihood @@ -0,0 +1 @@ +ab70f500cf24e876f6ae6bdc27525a1d6074fa9b6ea97770255d9fc2559b36ff \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-public_relations-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-public_relations-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..9ba711cca75cfa5f22bb2dc52e68839ac3820b88 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-public_relations-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-public_relations": {"acc": 0.3090909090909091, "acc_norm": 0.2636363636363636, "acc_norm_stderr": 0.04220224692971987, "acc_stderr": 0.044262946482000985}}, "versions": {"hendrycksTest-public_relations": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-security_studies-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-security_studies-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..6aa9b5ec005a326616b812b816b95329ad9349a2 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-security_studies-v0-loglikelihood @@ -0,0 +1 @@ +92dfffe2acf3278256486d3e1cf1edb5a739ad0a54c0f9c67695f7a411ed5f76 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-security_studies-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-security_studies-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..2c9de8886a29e0479074513470594c9266c5d0ac --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-security_studies-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-security_studies": {"acc": 0.2979591836734694, "acc_norm": 0.2693877551020408, "acc_norm_stderr": 0.02840125202902294, "acc_stderr": 0.029279567411065674}}, "versions": {"hendrycksTest-security_studies": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-sociology-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-sociology-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..d3f581c9f256191c2c0403a582fd72696150b34a --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-sociology-v0-loglikelihood @@ -0,0 +1 @@ +f99a3caece11169f2a5cc951001f92027104afd25d29b2a399883bd4bf118605 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-sociology-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-sociology-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..8711cf195e4fa92606a47c1b7c701643f0ef483e --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-sociology-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-sociology": {"acc": 0.23383084577114427, "acc_norm": 0.24875621890547264, "acc_norm_stderr": 0.030567675938916707, "acc_stderr": 0.02992941540834838}}, "versions": {"hendrycksTest-sociology": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-us_foreign_policy-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-us_foreign_policy-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..eed85dbaf98e67cc5cd0876bc5f73f3f9fb186fb --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-us_foreign_policy-v0-loglikelihood @@ -0,0 +1 @@ +a1a338d0083a21054f74d36a296d6bd8e2e457327c0fd630bebcc61ed758044d \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-us_foreign_policy-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-us_foreign_policy-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..1077380de88cb9ce23894ce31fbbeceea90f2079 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-us_foreign_policy-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-us_foreign_policy": {"acc": 0.2, "acc_norm": 0.24, "acc_norm_stderr": 0.04292346959909283, "acc_stderr": 0.040201512610368445}}, "versions": {"hendrycksTest-us_foreign_policy": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-virology-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-virology-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..3555c2c5351eb369bf0dc9cfedf93f0bbc3de7b4 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-virology-v0-loglikelihood @@ -0,0 +1 @@ +0ffa491f7bad2abbb64ecd752a295729167599b3815238cab0ecf4cb08bba9b6 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-virology-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-virology-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..0004b194049a5dce0266002b4a19882fbb8c6bfa --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-virology-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-virology": {"acc": 0.27710843373493976, "acc_norm": 0.2710843373493976, "acc_norm_stderr": 0.03460579907553027, "acc_stderr": 0.034843315926805875}}, "versions": {"hendrycksTest-virology": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-world_religions-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/hendrycksTest-world_religions-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..118c9b7435cc72387017ba1811d4bb62a16846b5 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-world_religions-v0-loglikelihood @@ -0,0 +1 @@ +97a0f68ba30ea3a6ef1db1a2925c964b09ecc54455a0a930da083e52677815bd \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/hendrycksTest-world_religions-v0-res.json b/lm-evaluation-harness/tests/testdata/hendrycksTest-world_religions-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..0fff75a7eaf2e0773a7e3dcda446f59a59dad878 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/hendrycksTest-world_religions-v0-res.json @@ -0,0 +1 @@ +{"results": {"hendrycksTest-world_religions": {"acc": 0.21637426900584794, "acc_norm": 0.22807017543859648, "acc_norm_stderr": 0.03218093795602357, "acc_stderr": 0.03158149539338734}}, "versions": {"hendrycksTest-world_religions": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/iwslt17-ar-en-v0-greedy_until b/lm-evaluation-harness/tests/testdata/iwslt17-ar-en-v0-greedy_until new file mode 100644 index 0000000000000000000000000000000000000000..82921d1db066020f53d61c21d46498a512144b37 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/iwslt17-ar-en-v0-greedy_until @@ -0,0 +1 @@ +e94d310de91fad7ce36f4cf3305552020221482c5588f2efcefaa019893504f1 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/iwslt17-ar-en-v0-res.json b/lm-evaluation-harness/tests/testdata/iwslt17-ar-en-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..0f414a928b62a8f8eefc939d693c944dd2521733 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/iwslt17-ar-en-v0-res.json @@ -0,0 +1 @@ +{"results": {"iwslt17-ar-en": {"bleu": 0.0, "bleu_stderr": 0.0, "chrf": 0.015049895477752772, "chrf_stderr": 0.0002940315671893584, "ter": 1.0, "ter_stderr": 0.0}}, "versions": {"iwslt17-ar-en": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/iwslt17-en-ar-v0-greedy_until b/lm-evaluation-harness/tests/testdata/iwslt17-en-ar-v0-greedy_until new file mode 100644 index 0000000000000000000000000000000000000000..fc59546576857b7f52dd4bfbdfc661c8ce871a6a --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/iwslt17-en-ar-v0-greedy_until @@ -0,0 +1 @@ +b20adbcd2c6d135e28600b427113532c5df624cb3a90e8c5e48715c09a3a38fa \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/iwslt17-en-ar-v0-res.json b/lm-evaluation-harness/tests/testdata/iwslt17-en-ar-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..a22fa9036c790cb48e142bd05a59da7824a9c83f --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/iwslt17-en-ar-v0-res.json @@ -0,0 +1 @@ +{"results": {"iwslt17-en-ar": {"bleu": 0.0, "bleu_stderr": 0.0, "chrf": 0.0, "chrf_stderr": 0.0, "ter": 1.0, "ter_stderr": 0.0}}, "versions": {"iwslt17-en-ar": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/lambada-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/lambada-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..efd450a8f2a4ca067f7380af809fdda48d1ee465 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/lambada-v0-loglikelihood @@ -0,0 +1 @@ +6829e6a8aa5922e6c92dd31403cc060f242dc0ede4a775e085a70da095ab2e20 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/lambada-v0-res.json b/lm-evaluation-harness/tests/testdata/lambada-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..ead0e9ce5d9629dea9be37e521fb3a152ced8680 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/lambada-v0-res.json @@ -0,0 +1 @@ +{"results": {"lambada": {"acc": 0.0, "acc_stderr": 0.0, "ppl": 1.6479047769869253, "ppl_stderr": 0.006497321146240192}}, "versions": {"lambada": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/lambada_cloze-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/lambada_cloze-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..b599a89f7af0c28e795e5c5dfc1961f34acde2fc --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/lambada_cloze-v0-loglikelihood @@ -0,0 +1 @@ +7655e748b63ae7e9911411d2d2a2577221d6c861ca4448509992541294d689f3 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/lambada_cloze-v0-res.json b/lm-evaluation-harness/tests/testdata/lambada_cloze-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..f3f3f931ac7e066cbab7b6ff68732360c764324f --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/lambada_cloze-v0-res.json @@ -0,0 +1 @@ +{"results": {"lambada_cloze": {"acc": 0.0, "acc_stderr": 0.0, "ppl": 1.6479047769869253, "ppl_stderr": 0.006497321146240192}}, "versions": {"lambada_cloze": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/lambada_mt_de-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/lambada_mt_de-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..ae19de0e6951bd90cd1e713d14816767496044e8 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/lambada_mt_de-v0-loglikelihood @@ -0,0 +1 @@ +5ad125e1708499832b2cee8c3388f89f9c0277010fd96fbd3359039ce8105984 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/lambada_mt_de-v0-res.json b/lm-evaluation-harness/tests/testdata/lambada_mt_de-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..7267ea739a54f5fd165ef0011d27446faac04689 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/lambada_mt_de-v0-res.json @@ -0,0 +1 @@ +{"results": {"lambada_mt_de": {"acc": 0.0, "acc_stderr": 0.0, "ppl": 1.6479047769869253, "ppl_stderr": 0.006497321146240192}}, "versions": {"lambada_mt_de": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/lambada_mt_en-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/lambada_mt_en-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..efd450a8f2a4ca067f7380af809fdda48d1ee465 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/lambada_mt_en-v0-loglikelihood @@ -0,0 +1 @@ +6829e6a8aa5922e6c92dd31403cc060f242dc0ede4a775e085a70da095ab2e20 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/lambada_mt_en-v0-res.json b/lm-evaluation-harness/tests/testdata/lambada_mt_en-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..561b88ffe110684b7de34a84ac613d1d901c72e5 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/lambada_mt_en-v0-res.json @@ -0,0 +1 @@ +{"results": {"lambada_mt_en": {"acc": 0.0, "acc_stderr": 0.0, "ppl": 1.6479047769869253, "ppl_stderr": 0.006497321146240192}}, "versions": {"lambada_mt_en": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/lambada_mt_es-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/lambada_mt_es-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..df895fe6d6bf04fc51c1633d26fb835941176534 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/lambada_mt_es-v0-loglikelihood @@ -0,0 +1 @@ +4a88f4b316c72fe0396c382d6cbb33568ac4d0ad225150d3536635c085359fc9 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/lambada_mt_es-v0-res.json b/lm-evaluation-harness/tests/testdata/lambada_mt_es-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..5f95957324e138bb424e71ff93f81a0c0a11f2cb --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/lambada_mt_es-v0-res.json @@ -0,0 +1 @@ +{"results": {"lambada_mt_es": {"acc": 0.0, "acc_stderr": 0.0, "ppl": 1.6479047769869253, "ppl_stderr": 0.006497321146240192}}, "versions": {"lambada_mt_es": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/lambada_mt_fr-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/lambada_mt_fr-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..3c444f66611959e4c13451d306fba403261ecfbb --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/lambada_mt_fr-v0-loglikelihood @@ -0,0 +1 @@ +5d16f4a0c51dc6d7b6df2ebeba2bbfa51e700b843779b559b3d90183d7b02a11 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/lambada_mt_fr-v0-res.json b/lm-evaluation-harness/tests/testdata/lambada_mt_fr-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..637c23500b9c153fe74ad9cb0369bd57f22d80a0 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/lambada_mt_fr-v0-res.json @@ -0,0 +1 @@ +{"results": {"lambada_mt_fr": {"acc": 0.0, "acc_stderr": 0.0, "ppl": 1.6479047769869253, "ppl_stderr": 0.006497321146240192}}, "versions": {"lambada_mt_fr": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/lambada_mt_it-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/lambada_mt_it-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..ca3fd80298aa1c565c978b26e992ccd42c7144f6 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/lambada_mt_it-v0-loglikelihood @@ -0,0 +1 @@ +fd87c6c5cf4e0499c5f9f80e5bd7ee6a4f3d2991902a0cc3ec9e6eaf22d6760a \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/lambada_mt_it-v0-res.json b/lm-evaluation-harness/tests/testdata/lambada_mt_it-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..b652210ae3df4694785f6bfe6543909435122dee --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/lambada_mt_it-v0-res.json @@ -0,0 +1 @@ +{"results": {"lambada_mt_it": {"acc": 0.0, "acc_stderr": 0.0, "ppl": 1.6479047769869253, "ppl_stderr": 0.006497321146240192}}, "versions": {"lambada_mt_it": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/lambada_openai-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/lambada_openai-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..efd450a8f2a4ca067f7380af809fdda48d1ee465 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/lambada_openai-v0-loglikelihood @@ -0,0 +1 @@ +6829e6a8aa5922e6c92dd31403cc060f242dc0ede4a775e085a70da095ab2e20 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/lambada_openai-v0-res.json b/lm-evaluation-harness/tests/testdata/lambada_openai-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..30fcb907b5dbbabb2af4cf3a156cf18c67d387df --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/lambada_openai-v0-res.json @@ -0,0 +1 @@ +{"results": {"lambada_openai": {"acc": 0.0, "acc_stderr": 0.0, "ppl": 1.6479047769869253, "ppl_stderr": 0.006497321146240192}}, "versions": {"lambada_openai": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/lambada_openai-v2.0-loglikelihood b/lm-evaluation-harness/tests/testdata/lambada_openai-v2.0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..688e67a5534f801d5b256905a0d05a60c0adf8fc --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/lambada_openai-v2.0-loglikelihood @@ -0,0 +1 @@ +9ca5643bbaafed2f027eab5b68cc438e9e268f6df9a678e956e61726a985cf0b \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/lambada_openai-v2.0-res.json b/lm-evaluation-harness/tests/testdata/lambada_openai-v2.0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..12e7f527bde7683ea74111603618c1e99cdd93a6 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/lambada_openai-v2.0-res.json @@ -0,0 +1 @@ +{"results": {"lambada_openai": {"acc": 0.0, "acc_stderr": 0.0, "ppl": 1.6479047769869253, "ppl_stderr": 0.006497321146240192}}, "versions": {"lambada_openai": "2.0"}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/lambada_openai_10_hf_pretrained-EleutherAI-pythia-14m-dtype-float32-device-cpu.txt b/lm-evaluation-harness/tests/testdata/lambada_openai_10_hf_pretrained-EleutherAI-pythia-14m-dtype-float32-device-cpu.txt new file mode 100644 index 0000000000000000000000000000000000000000..a3ea263b794b561212250ea3d962d1ba8eb89bde --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/lambada_openai_10_hf_pretrained-EleutherAI-pythia-14m-dtype-float32-device-cpu.txt @@ -0,0 +1,4 @@ +| Tasks |Version|Filter|n-shot| Metric | | Value | |Stderr| +|--------------|------:|------|-----:|----------|---|-------:|---|------| +|lambada_openai| 1|none | 0|acc |↑ | 0.1000|± | N/A| +| | |none | 0|perplexity|↓ |605.3866|± | N/A| \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/lambada_openai_cloze-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/lambada_openai_cloze-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..b599a89f7af0c28e795e5c5dfc1961f34acde2fc --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/lambada_openai_cloze-v0-loglikelihood @@ -0,0 +1 @@ +7655e748b63ae7e9911411d2d2a2577221d6c861ca4448509992541294d689f3 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/lambada_openai_cloze-v0-res.json b/lm-evaluation-harness/tests/testdata/lambada_openai_cloze-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..a52f2a9f1c83bcc119c95c05394f1bd2a86bf888 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/lambada_openai_cloze-v0-res.json @@ -0,0 +1 @@ +{"results": {"lambada_openai_cloze": {"acc": 0.0, "acc_stderr": 0.0, "ppl": 1.6479047769869253, "ppl_stderr": 0.006497321146240192}}, "versions": {"lambada_openai_cloze": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/lambada_openai_mt_de-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/lambada_openai_mt_de-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..ae19de0e6951bd90cd1e713d14816767496044e8 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/lambada_openai_mt_de-v0-loglikelihood @@ -0,0 +1 @@ +5ad125e1708499832b2cee8c3388f89f9c0277010fd96fbd3359039ce8105984 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/lambada_openai_mt_de-v0-res.json b/lm-evaluation-harness/tests/testdata/lambada_openai_mt_de-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..12f5d349ebd170ee5295656bc3907f872453eca6 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/lambada_openai_mt_de-v0-res.json @@ -0,0 +1 @@ +{"results": {"lambada_openai_mt_de": {"acc": 0.0, "acc_stderr": 0.0, "ppl": 1.6479047769869253, "ppl_stderr": 0.006497321146240192}}, "versions": {"lambada_openai_mt_de": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/lambada_openai_mt_en-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/lambada_openai_mt_en-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..efd450a8f2a4ca067f7380af809fdda48d1ee465 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/lambada_openai_mt_en-v0-loglikelihood @@ -0,0 +1 @@ +6829e6a8aa5922e6c92dd31403cc060f242dc0ede4a775e085a70da095ab2e20 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/lambada_openai_mt_en-v0-res.json b/lm-evaluation-harness/tests/testdata/lambada_openai_mt_en-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..f7fdfc9c2d5c6d5d4abb7d6e932454615c095ea1 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/lambada_openai_mt_en-v0-res.json @@ -0,0 +1 @@ +{"results": {"lambada_openai_mt_en": {"acc": 0.0, "acc_stderr": 0.0, "ppl": 1.6479047769869253, "ppl_stderr": 0.006497321146240192}}, "versions": {"lambada_openai_mt_en": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/lambada_openai_mt_es-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/lambada_openai_mt_es-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..df895fe6d6bf04fc51c1633d26fb835941176534 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/lambada_openai_mt_es-v0-loglikelihood @@ -0,0 +1 @@ +4a88f4b316c72fe0396c382d6cbb33568ac4d0ad225150d3536635c085359fc9 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/lambada_openai_mt_es-v0-res.json b/lm-evaluation-harness/tests/testdata/lambada_openai_mt_es-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..684e35a4cf44f85a0dc5f82fc06fb2b4ebc90316 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/lambada_openai_mt_es-v0-res.json @@ -0,0 +1 @@ +{"results": {"lambada_openai_mt_es": {"acc": 0.0, "acc_stderr": 0.0, "ppl": 1.6479047769869253, "ppl_stderr": 0.006497321146240192}}, "versions": {"lambada_openai_mt_es": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/lambada_openai_mt_fr-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/lambada_openai_mt_fr-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..3c444f66611959e4c13451d306fba403261ecfbb --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/lambada_openai_mt_fr-v0-loglikelihood @@ -0,0 +1 @@ +5d16f4a0c51dc6d7b6df2ebeba2bbfa51e700b843779b559b3d90183d7b02a11 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/lambada_openai_mt_fr-v0-res.json b/lm-evaluation-harness/tests/testdata/lambada_openai_mt_fr-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..ab6217943ab5d8e7547655c90ec95553c9557ee8 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/lambada_openai_mt_fr-v0-res.json @@ -0,0 +1 @@ +{"results": {"lambada_openai_mt_fr": {"acc": 0.0, "acc_stderr": 0.0, "ppl": 1.6479047769869253, "ppl_stderr": 0.006497321146240192}}, "versions": {"lambada_openai_mt_fr": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/lambada_openai_mt_it-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/lambada_openai_mt_it-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..ca3fd80298aa1c565c978b26e992ccd42c7144f6 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/lambada_openai_mt_it-v0-loglikelihood @@ -0,0 +1 @@ +fd87c6c5cf4e0499c5f9f80e5bd7ee6a4f3d2991902a0cc3ec9e6eaf22d6760a \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/lambada_openai_mt_it-v0-res.json b/lm-evaluation-harness/tests/testdata/lambada_openai_mt_it-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..2e7f6ef516e5e59af82f1768cfde132d57c1a1ec --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/lambada_openai_mt_it-v0-res.json @@ -0,0 +1 @@ +{"results": {"lambada_openai_mt_it": {"acc": 0.0, "acc_stderr": 0.0, "ppl": 1.6479047769869253, "ppl_stderr": 0.006497321146240192}}, "versions": {"lambada_openai_mt_it": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/lambada_standard-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/lambada_standard-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..fcbd56f50425ca6e143ccc0dd88458c051b63fb2 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/lambada_standard-v0-loglikelihood @@ -0,0 +1 @@ +8958d9f8d8145046b692fadd8a9cc9c8bad5617c10774280cf7c24c21d2be160 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/lambada_standard-v0-res.json b/lm-evaluation-harness/tests/testdata/lambada_standard-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..1f15d0be56b5edf18ad7cc2bec4977fae99f060b --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/lambada_standard-v0-res.json @@ -0,0 +1 @@ +{"results": {"lambada_standard": {"acc": 0.0, "acc_stderr": 0.0, "ppl": 1.6479047769869253, "ppl_stderr": 0.006497321146240192}}, "versions": {"lambada_standard": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/lambada_standard_cloze-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/lambada_standard_cloze-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..0a7b76241f898374a3a75952e16fe15af9a6d48e --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/lambada_standard_cloze-v0-loglikelihood @@ -0,0 +1 @@ +b604f00bc9f2a77ef41f8cfdb5a8509b3ae9266893b9e90abc665f5399ecba4e \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/lambada_standard_cloze-v0-res.json b/lm-evaluation-harness/tests/testdata/lambada_standard_cloze-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..5b3139fce60c4f456d1354e85f86f15657d63b85 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/lambada_standard_cloze-v0-res.json @@ -0,0 +1 @@ +{"results": {"lambada_standard_cloze": {"acc": 0.0, "acc_stderr": 0.0, "ppl": 1.6479047769869253, "ppl_stderr": 0.006497321146240192}}, "versions": {"lambada_standard_cloze": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/logiqa-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/logiqa-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..9cd40fce0a7062ec6897a119d44b1de88f762d08 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/logiqa-v0-loglikelihood @@ -0,0 +1 @@ +12495c50454ba5e1ce0753bd18c09aaca516bebd27648d815e37b15229dbf198 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/logiqa-v0-res.json b/lm-evaluation-harness/tests/testdata/logiqa-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..7a80c24d1b3e57ffca8ca89252d3c9b01b506f49 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/logiqa-v0-res.json @@ -0,0 +1 @@ +{"results": {"logiqa": {"acc": 0.25806451612903225, "acc_norm": 0.2764976958525346, "acc_norm_stderr": 0.017543209075825194, "acc_stderr": 0.017162894755127077}}, "versions": {"logiqa": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/math_algebra-v0-greedy_until b/lm-evaluation-harness/tests/testdata/math_algebra-v0-greedy_until new file mode 100644 index 0000000000000000000000000000000000000000..ce881a0232cff3f1025b746184ce8a0170e34303 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/math_algebra-v0-greedy_until @@ -0,0 +1 @@ +f19182ce697a2c095d9e5b56ee6659dc38c93994b69ca75d7c3d3f5fd87572b4 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/math_algebra-v0-res.json b/lm-evaluation-harness/tests/testdata/math_algebra-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..192cb9d8529cd67cb47e6f90d76a4a9e98b12d97 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/math_algebra-v0-res.json @@ -0,0 +1 @@ +{"results": {"math_algebra": {"acc": 0.0, "acc_stderr": 0.0}}, "versions": {"math_algebra": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/math_algebra-v1-greedy_until b/lm-evaluation-harness/tests/testdata/math_algebra-v1-greedy_until new file mode 100644 index 0000000000000000000000000000000000000000..ce881a0232cff3f1025b746184ce8a0170e34303 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/math_algebra-v1-greedy_until @@ -0,0 +1 @@ +f19182ce697a2c095d9e5b56ee6659dc38c93994b69ca75d7c3d3f5fd87572b4 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/math_algebra-v1-res.json b/lm-evaluation-harness/tests/testdata/math_algebra-v1-res.json new file mode 100644 index 0000000000000000000000000000000000000000..10d18c2f864117ae56fe56ba1191f6cde4bec7b3 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/math_algebra-v1-res.json @@ -0,0 +1 @@ +{"results": {"math_algebra": {"acc": 0.0, "acc_stderr": 0.0}}, "versions": {"math_algebra": 1}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/math_counting_and_prob-v0-greedy_until b/lm-evaluation-harness/tests/testdata/math_counting_and_prob-v0-greedy_until new file mode 100644 index 0000000000000000000000000000000000000000..6f49557ecf42758d64d1297c5569f3d4d95dd9c1 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/math_counting_and_prob-v0-greedy_until @@ -0,0 +1 @@ +2aa9ae43ee9dbb2457525247d7b65358632c5eaa9cbfc40cf95a4f17f5d942ad \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/math_counting_and_prob-v0-res.json b/lm-evaluation-harness/tests/testdata/math_counting_and_prob-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..8ee1d031de8ec7d2af61c83567d433f9116ba24d --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/math_counting_and_prob-v0-res.json @@ -0,0 +1 @@ +{"results": {"math_counting_and_prob": {"acc": 0.0, "acc_stderr": 0.0}}, "versions": {"math_counting_and_prob": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/math_counting_and_prob-v1-greedy_until b/lm-evaluation-harness/tests/testdata/math_counting_and_prob-v1-greedy_until new file mode 100644 index 0000000000000000000000000000000000000000..6f49557ecf42758d64d1297c5569f3d4d95dd9c1 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/math_counting_and_prob-v1-greedy_until @@ -0,0 +1 @@ +2aa9ae43ee9dbb2457525247d7b65358632c5eaa9cbfc40cf95a4f17f5d942ad \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/math_counting_and_prob-v1-res.json b/lm-evaluation-harness/tests/testdata/math_counting_and_prob-v1-res.json new file mode 100644 index 0000000000000000000000000000000000000000..240f7b6b42b77b8e94c1ec2eab2df808181a2cb3 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/math_counting_and_prob-v1-res.json @@ -0,0 +1 @@ +{"results": {"math_counting_and_prob": {"acc": 0.0, "acc_stderr": 0.0}}, "versions": {"math_counting_and_prob": 1}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/math_geometry-v0-greedy_until b/lm-evaluation-harness/tests/testdata/math_geometry-v0-greedy_until new file mode 100644 index 0000000000000000000000000000000000000000..1c7362fe44e4432f56f18932b4b429d5cf573399 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/math_geometry-v0-greedy_until @@ -0,0 +1 @@ +46bc4cb219b6903397da782699a684bdbb982c0c954ff82e6beeed5c84878f42 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/math_geometry-v0-res.json b/lm-evaluation-harness/tests/testdata/math_geometry-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..1b25dc283c96c63d30df9f0ce3d04aadb8f93625 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/math_geometry-v0-res.json @@ -0,0 +1 @@ +{"results": {"math_geometry": {"acc": 0.0, "acc_stderr": 0.0}}, "versions": {"math_geometry": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/math_geometry-v1-greedy_until b/lm-evaluation-harness/tests/testdata/math_geometry-v1-greedy_until new file mode 100644 index 0000000000000000000000000000000000000000..1c7362fe44e4432f56f18932b4b429d5cf573399 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/math_geometry-v1-greedy_until @@ -0,0 +1 @@ +46bc4cb219b6903397da782699a684bdbb982c0c954ff82e6beeed5c84878f42 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/math_geometry-v1-res.json b/lm-evaluation-harness/tests/testdata/math_geometry-v1-res.json new file mode 100644 index 0000000000000000000000000000000000000000..eb6851fc63ff08c657743ef6abf5073ba73144e5 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/math_geometry-v1-res.json @@ -0,0 +1 @@ +{"results": {"math_geometry": {"acc": 0.0, "acc_stderr": 0.0}}, "versions": {"math_geometry": 1}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/math_intermediate_algebra-v0-greedy_until b/lm-evaluation-harness/tests/testdata/math_intermediate_algebra-v0-greedy_until new file mode 100644 index 0000000000000000000000000000000000000000..3ab10de26a038019a18699e20887de6da66981c4 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/math_intermediate_algebra-v0-greedy_until @@ -0,0 +1 @@ +d53c699de272d517ed7ad783b4e692302be9f9f97a8d4ac7a6541e538a7cabe0 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/math_intermediate_algebra-v0-res.json b/lm-evaluation-harness/tests/testdata/math_intermediate_algebra-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..7a195d9ac43e6feb4a7fc354f5dc424a27b0bf7d --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/math_intermediate_algebra-v0-res.json @@ -0,0 +1 @@ +{"results": {"math_intermediate_algebra": {"acc": 0.0, "acc_stderr": 0.0}}, "versions": {"math_intermediate_algebra": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/math_intermediate_algebra-v1-greedy_until b/lm-evaluation-harness/tests/testdata/math_intermediate_algebra-v1-greedy_until new file mode 100644 index 0000000000000000000000000000000000000000..3ab10de26a038019a18699e20887de6da66981c4 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/math_intermediate_algebra-v1-greedy_until @@ -0,0 +1 @@ +d53c699de272d517ed7ad783b4e692302be9f9f97a8d4ac7a6541e538a7cabe0 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/math_intermediate_algebra-v1-res.json b/lm-evaluation-harness/tests/testdata/math_intermediate_algebra-v1-res.json new file mode 100644 index 0000000000000000000000000000000000000000..63ab45b9ff890a0ef7c2108133b23bf0043f13f8 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/math_intermediate_algebra-v1-res.json @@ -0,0 +1 @@ +{"results": {"math_intermediate_algebra": {"acc": 0.0, "acc_stderr": 0.0}}, "versions": {"math_intermediate_algebra": 1}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/math_num_theory-v0-greedy_until b/lm-evaluation-harness/tests/testdata/math_num_theory-v0-greedy_until new file mode 100644 index 0000000000000000000000000000000000000000..82febb9f5dfeefbd6dc5d244574ac5666c6b8bba --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/math_num_theory-v0-greedy_until @@ -0,0 +1 @@ +b920ccb507afdcf3ef6f4c04891913731e9f32ec914801791c6d9f8abf6e1897 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/math_num_theory-v0-res.json b/lm-evaluation-harness/tests/testdata/math_num_theory-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..a27a38fa9d4f3a924828bdb4526953a35328c7e5 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/math_num_theory-v0-res.json @@ -0,0 +1 @@ +{"results": {"math_num_theory": {"acc": 0.0, "acc_stderr": 0.0}}, "versions": {"math_num_theory": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/math_num_theory-v1-greedy_until b/lm-evaluation-harness/tests/testdata/math_num_theory-v1-greedy_until new file mode 100644 index 0000000000000000000000000000000000000000..82febb9f5dfeefbd6dc5d244574ac5666c6b8bba --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/math_num_theory-v1-greedy_until @@ -0,0 +1 @@ +b920ccb507afdcf3ef6f4c04891913731e9f32ec914801791c6d9f8abf6e1897 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/math_num_theory-v1-res.json b/lm-evaluation-harness/tests/testdata/math_num_theory-v1-res.json new file mode 100644 index 0000000000000000000000000000000000000000..00917b90ddb0602c62c8a9fef959b9e91eb45c2e --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/math_num_theory-v1-res.json @@ -0,0 +1 @@ +{"results": {"math_num_theory": {"acc": 0.0, "acc_stderr": 0.0}}, "versions": {"math_num_theory": 1}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/math_prealgebra-v0-greedy_until b/lm-evaluation-harness/tests/testdata/math_prealgebra-v0-greedy_until new file mode 100644 index 0000000000000000000000000000000000000000..5200f4cfa9ed3a735661e987791bf1434555db6e --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/math_prealgebra-v0-greedy_until @@ -0,0 +1 @@ +752cdf343d7152e476b0273065024f6ea0e0f47ea385c6bdf9067736cb39724a \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/math_prealgebra-v0-res.json b/lm-evaluation-harness/tests/testdata/math_prealgebra-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..b3ada8a6be4d86b71a0c6b92c605d3c8a25a29a2 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/math_prealgebra-v0-res.json @@ -0,0 +1 @@ +{"results": {"math_prealgebra": {"acc": 0.0, "acc_stderr": 0.0}}, "versions": {"math_prealgebra": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/math_prealgebra-v1-greedy_until b/lm-evaluation-harness/tests/testdata/math_prealgebra-v1-greedy_until new file mode 100644 index 0000000000000000000000000000000000000000..5200f4cfa9ed3a735661e987791bf1434555db6e --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/math_prealgebra-v1-greedy_until @@ -0,0 +1 @@ +752cdf343d7152e476b0273065024f6ea0e0f47ea385c6bdf9067736cb39724a \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/math_prealgebra-v1-res.json b/lm-evaluation-harness/tests/testdata/math_prealgebra-v1-res.json new file mode 100644 index 0000000000000000000000000000000000000000..e3869faa8012568df2eae14e3774712960c4a544 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/math_prealgebra-v1-res.json @@ -0,0 +1 @@ +{"results": {"math_prealgebra": {"acc": 0.0, "acc_stderr": 0.0}}, "versions": {"math_prealgebra": 1}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/math_precalc-v0-greedy_until b/lm-evaluation-harness/tests/testdata/math_precalc-v0-greedy_until new file mode 100644 index 0000000000000000000000000000000000000000..71bbd8d9c221ca484d517bda46c109b2610f79f6 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/math_precalc-v0-greedy_until @@ -0,0 +1 @@ +bc834b06fd79473ca6fe38a51b714aad0bf0478c1b0eec787eca34dbdf69cb71 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/math_precalc-v0-res.json b/lm-evaluation-harness/tests/testdata/math_precalc-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..699dc5fe38ea411d6d53c9e19d78ba6d96ddfb40 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/math_precalc-v0-res.json @@ -0,0 +1 @@ +{"results": {"math_precalc": {"acc": 0.0, "acc_stderr": 0.0}}, "versions": {"math_precalc": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/math_precalc-v1-greedy_until b/lm-evaluation-harness/tests/testdata/math_precalc-v1-greedy_until new file mode 100644 index 0000000000000000000000000000000000000000..71bbd8d9c221ca484d517bda46c109b2610f79f6 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/math_precalc-v1-greedy_until @@ -0,0 +1 @@ +bc834b06fd79473ca6fe38a51b714aad0bf0478c1b0eec787eca34dbdf69cb71 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/math_precalc-v1-res.json b/lm-evaluation-harness/tests/testdata/math_precalc-v1-res.json new file mode 100644 index 0000000000000000000000000000000000000000..a5846590a3b28f2382d00a3400e1c46a9018adea --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/math_precalc-v1-res.json @@ -0,0 +1 @@ +{"results": {"math_precalc": {"acc": 0.0, "acc_stderr": 0.0}}, "versions": {"math_precalc": 1}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/mathqa-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/mathqa-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..9f33d79035cc7caf00704a8764aa0adf657c0b78 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/mathqa-v0-loglikelihood @@ -0,0 +1 @@ +a45260e49f02c7cb8886b3746db4d388890860b202dd8a9f0267e3c324e0af13 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/mathqa-v0-res.json b/lm-evaluation-harness/tests/testdata/mathqa-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..dabd07c07cbbad2886b20acb25189b111676bbcd --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/mathqa-v0-res.json @@ -0,0 +1 @@ +{"results": {"mathqa": {"acc": 0.20770519262981574, "acc_norm": 0.2050251256281407, "acc_norm_stderr": 0.007390619359738901, "acc_stderr": 0.007426217631188539}}, "versions": {"mathqa": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/mc_taco-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/mc_taco-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..f0ce5c64580d1132710e596cc287126ba77394e6 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/mc_taco-v0-loglikelihood @@ -0,0 +1 @@ +1811808ef05afd5f30ffc3471622a3dd7a1b681b17a2f7616695ad6b2a45943c \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/mc_taco-v0-res.json b/lm-evaluation-harness/tests/testdata/mc_taco-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..fc36d1ed3ff4d02330a13eb7431d5413b4c484e5 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/mc_taco-v0-res.json @@ -0,0 +1 @@ +{"results": {"mc_taco": {"em": 0.07732732732732733, "f1": 0.41600515965511614}}, "versions": {"mc_taco": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/mmlu_stem_10_hf_pretrained-EleutherAI-pythia-14m-dtype-float32-device-cpu.txt b/lm-evaluation-harness/tests/testdata/mmlu_stem_10_hf_pretrained-EleutherAI-pythia-14m-dtype-float32-device-cpu.txt new file mode 100644 index 0000000000000000000000000000000000000000..8c6c56c1261f69f89dff5ac169fd75d16a5a9f43 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/mmlu_stem_10_hf_pretrained-EleutherAI-pythia-14m-dtype-float32-device-cpu.txt @@ -0,0 +1,22 @@ +| Tasks |Version|Filter|n-shot|Metric| |Value | |Stderr| +|-------------------------------|------:|------|-----:|------|---|-----:|---|------| +|stem | 2|none | |acc |↑ |0.2474|± | N/A| +| - abstract_algebra | 1|none | 0|acc |↑ |0.2000|± | N/A| +| - anatomy | 1|none | 0|acc |↑ |0.3000|± | N/A| +| - astronomy | 1|none | 0|acc |↑ |0.1000|± | N/A| +| - college_biology | 1|none | 0|acc |↑ |0.3000|± | N/A| +| - college_chemistry | 1|none | 0|acc |↑ |0.1000|± | N/A| +| - college_computer_science | 1|none | 0|acc |↑ |0.2000|± | N/A| +| - college_mathematics | 1|none | 0|acc |↑ |0.2000|± | N/A| +| - college_physics | 1|none | 0|acc |↑ |0.3000|± | N/A| +| - computer_security | 1|none | 0|acc |↑ |0.5000|± | N/A| +| - conceptual_physics | 1|none | 0|acc |↑ |0.3000|± | N/A| +| - electrical_engineering | 1|none | 0|acc |↑ |0.4000|± | N/A| +| - elementary_mathematics | 1|none | 0|acc |↑ |0.0000|± | N/A| +| - high_school_biology | 1|none | 0|acc |↑ |0.3000|± | N/A| +| - high_school_chemistry | 1|none | 0|acc |↑ |0.4000|± | N/A| +| - high_school_computer_science| 1|none | 0|acc |↑ |0.3000|± | N/A| +| - high_school_mathematics | 1|none | 0|acc |↑ |0.2000|± | N/A| +| - high_school_physics | 1|none | 0|acc |↑ |0.3000|± | N/A| +| - high_school_statistics | 1|none | 0|acc |↑ |0.0000|± | N/A| +| - machine_learning | 1|none | 0|acc |↑ |0.3000|± | N/A| \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/mnli-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/mnli-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..433b76d01094a18991412513044f0933eb0bf3f5 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/mnli-v0-loglikelihood @@ -0,0 +1 @@ +4fc7b56b8f1e37e38f4a052b227baec2df914c898c3405d3e994726ba4fba976 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/mnli-v0-res.json b/lm-evaluation-harness/tests/testdata/mnli-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..d9dada7a0244534c35d86efb71a03fbd90217328 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/mnli-v0-res.json @@ -0,0 +1 @@ +{"results": {"mnli": {"acc": 0.32868059093224655, "acc_stderr": 0.004741640290753859}}, "versions": {"mnli": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/mnli_mismatched-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/mnli_mismatched-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..3fb242da3a2d274cbcc84bf86a6bb11f02df27ab --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/mnli_mismatched-v0-loglikelihood @@ -0,0 +1 @@ +3784acf322e79f31702a7a0612030e4ba5c4fc466ad976a34ee3f3d7278c01f0 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/mnli_mismatched-v0-res.json b/lm-evaluation-harness/tests/testdata/mnli_mismatched-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..261deed96275da1af0c8a0616b0af6247cfaf1c0 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/mnli_mismatched-v0-res.json @@ -0,0 +1 @@ +{"results": {"mnli_mismatched": {"acc": 0.3360455655004068, "acc_stderr": 0.004763973908606819}}, "versions": {"mnli_mismatched": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/mrpc-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/mrpc-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..95c849a153762819da1ce59c1b58a2013b97ef6a --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/mrpc-v0-loglikelihood @@ -0,0 +1 @@ +9f54cbff8d6accba99cfa2c4c4b359563313941018173d7dcf9e32dc28c06583 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/mrpc-v0-res.json b/lm-evaluation-harness/tests/testdata/mrpc-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..f141eaa0a49aceaae493aea7080eab4e8b1cec16 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/mrpc-v0-res.json @@ -0,0 +1 @@ +{"results": {"mrpc": {"acc": 0.5392156862745098, "acc_stderr": 0.024707732873723128, "f1": 0.5982905982905982, "f1_stderr": 0.028928325246283727}}, "versions": {"mrpc": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/multirc-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/multirc-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..b3681ec17595adc4c4541ded263add219912af58 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/multirc-v0-loglikelihood @@ -0,0 +1 @@ +cdb026c027437a8b4653212d0944d36fc16f49921dcb8e4bef899d15a55e9f80 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/multirc-v0-res.json b/lm-evaluation-harness/tests/testdata/multirc-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..87e9c532eb7d7deb7d08635dd955df7a68ab9813 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/multirc-v0-res.json @@ -0,0 +1 @@ +{"results": {"multirc": {"acc": 0.07450157397691501, "acc_stderr": 0.008510441526175931}}, "versions": {"multirc": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/multirc-v1-loglikelihood b/lm-evaluation-harness/tests/testdata/multirc-v1-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..52a89c6f9eaf1cece362cf2d4bd114f8ae3cbdda --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/multirc-v1-loglikelihood @@ -0,0 +1 @@ +0e793bd6f637a70a04c6f2cda080188fc037961b2f909095fe63f7bdbc4a90c6 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/multirc-v1-res.json b/lm-evaluation-harness/tests/testdata/multirc-v1-res.json new file mode 100644 index 0000000000000000000000000000000000000000..938141bbb888f55c3aa2786868c28925ac3fd123 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/multirc-v1-res.json @@ -0,0 +1 @@ +{"results": {"multirc": {"acc": 0.046169989506820566, "acc_stderr": 0.006801377886208738}}, "versions": {"multirc": 1}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/mutual-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/mutual-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..0022f466d25f3e3a639720e4600732c9c0c1141d --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/mutual-v0-loglikelihood @@ -0,0 +1 @@ +f759213a28f0412510bf1a24c9cab0dae64bdee902d42a26225295445e7779db \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/mutual-v0-res.json b/lm-evaluation-harness/tests/testdata/mutual-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..2d240576b3b8d891ff91a47770df9990edf34105 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/mutual-v0-res.json @@ -0,0 +1 @@ +{"results": {"mutual": {"mrr": 0.5023513920240772, "mrr_stderr": 0.009501864812936679, "r@1": 0.22573363431151242, "r@1_stderr": 0.014053085820407457, "r@2": 0.4221218961625282, "r@2_stderr": 0.016602191705517556}}, "versions": {"mutual": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/mutual-v1-loglikelihood b/lm-evaluation-harness/tests/testdata/mutual-v1-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..0022f466d25f3e3a639720e4600732c9c0c1141d --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/mutual-v1-loglikelihood @@ -0,0 +1 @@ +f759213a28f0412510bf1a24c9cab0dae64bdee902d42a26225295445e7779db \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/mutual-v1-res.json b/lm-evaluation-harness/tests/testdata/mutual-v1-res.json new file mode 100644 index 0000000000000000000000000000000000000000..42e97c6f1a1b65ef76cc3941c8b08e8ca836a59c --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/mutual-v1-res.json @@ -0,0 +1 @@ +{"results": {"mutual": {"mrr": 0.5023513920240772, "mrr_stderr": 0.009501864812936679, "r@1": 0.22460496613995484, "r@1_stderr": 0.014028122493992806, "r@2": 0.4706546275395034, "r@2_stderr": 0.016778343895001414}}, "versions": {"mutual": 1}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/mutual_plus-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/mutual_plus-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..f4ba9d37310a19cc7928fd0d599776d8a9da8dba --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/mutual_plus-v0-loglikelihood @@ -0,0 +1 @@ +b846bb9db109535f59a93d1ce340cf09f68bdf4fed5b8decd168784220fe07fa \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/mutual_plus-v0-res.json b/lm-evaluation-harness/tests/testdata/mutual_plus-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..9c0348826363dfe77e60cb28cf546110e25bab35 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/mutual_plus-v0-res.json @@ -0,0 +1 @@ +{"results": {"mutual_plus": {"mrr": 0.5275583145221953, "mrr_stderr": 0.009940894824430708, "r@1": 0.2595936794582393, "r@1_stderr": 0.014737047402750955, "r@2": 0.45372460496614, "r@2_stderr": 0.01673517854461967}}, "versions": {"mutual_plus": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/mutual_plus-v1-loglikelihood b/lm-evaluation-harness/tests/testdata/mutual_plus-v1-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..f4ba9d37310a19cc7928fd0d599776d8a9da8dba --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/mutual_plus-v1-loglikelihood @@ -0,0 +1 @@ +b846bb9db109535f59a93d1ce340cf09f68bdf4fed5b8decd168784220fe07fa \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/mutual_plus-v1-res.json b/lm-evaluation-harness/tests/testdata/mutual_plus-v1-res.json new file mode 100644 index 0000000000000000000000000000000000000000..cdb6c85b65643b2214358d18b057d0737d53b9ba --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/mutual_plus-v1-res.json @@ -0,0 +1 @@ +{"results": {"mutual_plus": {"mrr": 0.5275583145221953, "mrr_stderr": 0.009940894824430708, "r@1": 0.26297968397291194, "r@1_stderr": 0.01479889176605113, "r@2": 0.5, "r@2_stderr": 0.01680731613632036}}, "versions": {"mutual_plus": 1}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/openbookqa-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/openbookqa-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..b2cc5e9795fd1623bfc11e4d1cb53b0e1baa3dbf --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/openbookqa-v0-loglikelihood @@ -0,0 +1 @@ +78a49a0ca1a47373adb33463b1d092e6bc0d8f4b01bcb380ada48065037849d7 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/openbookqa-v0-res.json b/lm-evaluation-harness/tests/testdata/openbookqa-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..04f4c25442e678a63d3f6213dc9364bfa25b1a7a --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/openbookqa-v0-res.json @@ -0,0 +1 @@ +{"results": {"openbookqa": {"acc": 0.214, "acc_norm": 0.276, "acc_norm_stderr": 0.020011219298073517, "acc_stderr": 0.018359797502387046}}, "versions": {"openbookqa": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_arxiv-v0-loglikelihood_rolling b/lm-evaluation-harness/tests/testdata/pile_arxiv-v0-loglikelihood_rolling new file mode 100644 index 0000000000000000000000000000000000000000..3aa1d8c7349449271fbd81fbbc06fde47a116028 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_arxiv-v0-loglikelihood_rolling @@ -0,0 +1 @@ +814f9954e44368559602c00f7e85fa3971acdfd0315f508ec7df6318a79c55ec \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_arxiv-v0-res.json b/lm-evaluation-harness/tests/testdata/pile_arxiv-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..d19d0c6fee7f47af1ad3f5af9ff1d7a1544e2e98 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_arxiv-v0-res.json @@ -0,0 +1 @@ +{"results": {"pile_arxiv": {"bits_per_byte": 1.0750412350569374e-05, "byte_perplexity": 1.0000107504701365, "word_perplexity": 1.0000819333090385}}, "versions": {"pile_arxiv": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_arxiv-v1-loglikelihood_rolling b/lm-evaluation-harness/tests/testdata/pile_arxiv-v1-loglikelihood_rolling new file mode 100644 index 0000000000000000000000000000000000000000..3aa1d8c7349449271fbd81fbbc06fde47a116028 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_arxiv-v1-loglikelihood_rolling @@ -0,0 +1 @@ +814f9954e44368559602c00f7e85fa3971acdfd0315f508ec7df6318a79c55ec \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_arxiv-v1-res.json b/lm-evaluation-harness/tests/testdata/pile_arxiv-v1-res.json new file mode 100644 index 0000000000000000000000000000000000000000..05cbab38732c94665750aac31cd2c41688552a8d --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_arxiv-v1-res.json @@ -0,0 +1 @@ +{"results": {"pile_arxiv": {"bits_per_byte": 1.55095665856779e-05, "byte_perplexity": 1.0000107504701365, "word_perplexity": 1.0000819333090385}}, "versions": {"pile_arxiv": 1}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_bookcorpus2-v0-loglikelihood_rolling b/lm-evaluation-harness/tests/testdata/pile_bookcorpus2-v0-loglikelihood_rolling new file mode 100644 index 0000000000000000000000000000000000000000..b37a91cc2dea829e8dab7bb0fe934442c54b3a26 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_bookcorpus2-v0-loglikelihood_rolling @@ -0,0 +1 @@ +5c17ddfebeab8c41dabadb6fc216ceda91e3fe5dc95aaf1b2c843d7f11828b03 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_bookcorpus2-v0-res.json b/lm-evaluation-harness/tests/testdata/pile_bookcorpus2-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..698b03e8b3b437f94f22744ffe12ba2fff9285f6 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_bookcorpus2-v0-res.json @@ -0,0 +1 @@ +{"results": {"pile_bookcorpus2": {"bits_per_byte": 1.1631037706429144e-06, "byte_perplexity": 1.000001163104447, "word_perplexity": 1.0000066499426599}}, "versions": {"pile_bookcorpus2": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_bookcorpus2-v1-loglikelihood_rolling b/lm-evaluation-harness/tests/testdata/pile_bookcorpus2-v1-loglikelihood_rolling new file mode 100644 index 0000000000000000000000000000000000000000..b37a91cc2dea829e8dab7bb0fe934442c54b3a26 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_bookcorpus2-v1-loglikelihood_rolling @@ -0,0 +1 @@ +5c17ddfebeab8c41dabadb6fc216ceda91e3fe5dc95aaf1b2c843d7f11828b03 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_bookcorpus2-v1-res.json b/lm-evaluation-harness/tests/testdata/pile_bookcorpus2-v1-res.json new file mode 100644 index 0000000000000000000000000000000000000000..967c14934b81e0880063c4239593fb74cd99cd8d --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_bookcorpus2-v1-res.json @@ -0,0 +1 @@ +{"results": {"pile_bookcorpus2": {"bits_per_byte": 1.6780040419457868e-06, "byte_perplexity": 1.000001163104447, "word_perplexity": 1.0000066499426599}}, "versions": {"pile_bookcorpus2": 1}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_books3-v0-loglikelihood_rolling b/lm-evaluation-harness/tests/testdata/pile_books3-v0-loglikelihood_rolling new file mode 100644 index 0000000000000000000000000000000000000000..b483d3b45b43abddd6cbd169a8afda8d3f803d9c --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_books3-v0-loglikelihood_rolling @@ -0,0 +1 @@ +0f8f36f705b999b6d55fa72ff89a82793dd1cb568ab1f8727a6a2086a12b9410 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_books3-v0-res.json b/lm-evaluation-harness/tests/testdata/pile_books3-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..df19cd0a18f122d695f8aea4a717ab4dde79a987 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_books3-v0-res.json @@ -0,0 +1 @@ +{"results": {"pile_books3": {"bits_per_byte": 8.942486206275221e-07, "byte_perplexity": 1.0000008942490204, "word_perplexity": 1.0000052870063607}}, "versions": {"pile_books3": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_books3-v1-loglikelihood_rolling b/lm-evaluation-harness/tests/testdata/pile_books3-v1-loglikelihood_rolling new file mode 100644 index 0000000000000000000000000000000000000000..b483d3b45b43abddd6cbd169a8afda8d3f803d9c --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_books3-v1-loglikelihood_rolling @@ -0,0 +1 @@ +0f8f36f705b999b6d55fa72ff89a82793dd1cb568ab1f8727a6a2086a12b9410 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_books3-v1-res.json b/lm-evaluation-harness/tests/testdata/pile_books3-v1-res.json new file mode 100644 index 0000000000000000000000000000000000000000..6ff7a517112eba76e15e999e9974124e04f07a83 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_books3-v1-res.json @@ -0,0 +1 @@ +{"results": {"pile_books3": {"bits_per_byte": 1.2901280503011222e-06, "byte_perplexity": 1.0000008942490204, "word_perplexity": 1.0000052870063607}}, "versions": {"pile_books3": 1}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_dm-mathematics-v0-loglikelihood_rolling b/lm-evaluation-harness/tests/testdata/pile_dm-mathematics-v0-loglikelihood_rolling new file mode 100644 index 0000000000000000000000000000000000000000..2fb27786c54abe6303683c0a247d4c689586a97c --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_dm-mathematics-v0-loglikelihood_rolling @@ -0,0 +1 @@ +d5b7967c0ece8b816f3921a8bd0fad23365349e935b491595e2ad1135af42da6 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_dm-mathematics-v0-res.json b/lm-evaluation-harness/tests/testdata/pile_dm-mathematics-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..860aa06c974e58d03f54ab1d9cb14c7e98019d4e --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_dm-mathematics-v0-res.json @@ -0,0 +1 @@ +{"results": {"pile_dm-mathematics": {"bits_per_byte": 6.176600873627999e-05, "byte_perplexity": 1.0000617679162955, "word_perplexity": 1.0002875035042451}}, "versions": {"pile_dm-mathematics": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_dm-mathematics-v1-loglikelihood_rolling b/lm-evaluation-harness/tests/testdata/pile_dm-mathematics-v1-loglikelihood_rolling new file mode 100644 index 0000000000000000000000000000000000000000..2fb27786c54abe6303683c0a247d4c689586a97c --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_dm-mathematics-v1-loglikelihood_rolling @@ -0,0 +1 @@ +d5b7967c0ece8b816f3921a8bd0fad23365349e935b491595e2ad1135af42da6 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_dm-mathematics-v1-res.json b/lm-evaluation-harness/tests/testdata/pile_dm-mathematics-v1-res.json new file mode 100644 index 0000000000000000000000000000000000000000..192e9066a42acf28436ae325a212b2a7c2ebf517 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_dm-mathematics-v1-res.json @@ -0,0 +1 @@ +{"results": {"pile_dm-mathematics": {"bits_per_byte": 8.910951449933553e-05, "byte_perplexity": 1.0000617679162955, "word_perplexity": 1.0002875035042451}}, "versions": {"pile_dm-mathematics": 1}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_enron-v0-loglikelihood_rolling b/lm-evaluation-harness/tests/testdata/pile_enron-v0-loglikelihood_rolling new file mode 100644 index 0000000000000000000000000000000000000000..57dbe764605ef5e1e4578682549a001c851704c0 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_enron-v0-loglikelihood_rolling @@ -0,0 +1 @@ +4baa6ccdc9e3aa9921675ab4400d5e89d7b546b844a8ea28f6461d649066418a \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_enron-v0-res.json b/lm-evaluation-harness/tests/testdata/pile_enron-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..a4a49493d56db35c99b7e58ea66ebc21304184b2 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_enron-v0-res.json @@ -0,0 +1 @@ +{"results": {"pile_enron": {"bits_per_byte": 0.0003163902828673244, "byte_perplexity": 1.000316440339552, "word_perplexity": 1.00224668051869}}, "versions": {"pile_enron": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_enron-v1-loglikelihood_rolling b/lm-evaluation-harness/tests/testdata/pile_enron-v1-loglikelihood_rolling new file mode 100644 index 0000000000000000000000000000000000000000..57dbe764605ef5e1e4578682549a001c851704c0 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_enron-v1-loglikelihood_rolling @@ -0,0 +1 @@ +4baa6ccdc9e3aa9921675ab4400d5e89d7b546b844a8ea28f6461d649066418a \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_enron-v1-res.json b/lm-evaluation-harness/tests/testdata/pile_enron-v1-res.json new file mode 100644 index 0000000000000000000000000000000000000000..abe7b45f9aff9b6427068ceb1ba39977fa843c38 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_enron-v1-res.json @@ -0,0 +1 @@ +{"results": {"pile_enron": {"bits_per_byte": 0.0004564546920781453, "byte_perplexity": 1.000316440339552, "word_perplexity": 1.00224668051869}}, "versions": {"pile_enron": 1}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_europarl-v0-loglikelihood_rolling b/lm-evaluation-harness/tests/testdata/pile_europarl-v0-loglikelihood_rolling new file mode 100644 index 0000000000000000000000000000000000000000..80272607557f6e0c97220efa30c8b9ad38f52aa8 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_europarl-v0-loglikelihood_rolling @@ -0,0 +1 @@ +e67d3dbccd47d308bfc5b0e66b76d0dfc5e386ebfa94e056562c2281c395543f \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_europarl-v0-res.json b/lm-evaluation-harness/tests/testdata/pile_europarl-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..4c53edd2ce620475c056ccca5cde73380c246074 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_europarl-v0-res.json @@ -0,0 +1 @@ +{"results": {"pile_europarl": {"bits_per_byte": 8.648858203555344e-06, "byte_perplexity": 1.000008648895605, "word_perplexity": 1.000063506523818}}, "versions": {"pile_europarl": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_europarl-v1-loglikelihood_rolling b/lm-evaluation-harness/tests/testdata/pile_europarl-v1-loglikelihood_rolling new file mode 100644 index 0000000000000000000000000000000000000000..80272607557f6e0c97220efa30c8b9ad38f52aa8 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_europarl-v1-loglikelihood_rolling @@ -0,0 +1 @@ +e67d3dbccd47d308bfc5b0e66b76d0dfc5e386ebfa94e056562c2281c395543f \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_europarl-v1-res.json b/lm-evaluation-harness/tests/testdata/pile_europarl-v1-res.json new file mode 100644 index 0000000000000000000000000000000000000000..b948f0d3691443f50c9f9d5ae24804b0c7e79aaa --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_europarl-v1-res.json @@ -0,0 +1 @@ +{"results": {"pile_europarl": {"bits_per_byte": 1.2477664839621123e-05, "byte_perplexity": 1.000008648895605, "word_perplexity": 1.000063506523818}}, "versions": {"pile_europarl": 1}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_freelaw-v0-loglikelihood_rolling b/lm-evaluation-harness/tests/testdata/pile_freelaw-v0-loglikelihood_rolling new file mode 100644 index 0000000000000000000000000000000000000000..7b5771f4911f3069217d75d12cbdfa1a579b6663 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_freelaw-v0-loglikelihood_rolling @@ -0,0 +1 @@ +d77f3f68aadd6cbf1290c2f6737b2ed5d5c2a60e4c81a65c280f207783caabe1 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_freelaw-v0-res.json b/lm-evaluation-harness/tests/testdata/pile_freelaw-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..0bda41ffb37dd04bebd9982faf464616dd82a31d --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_freelaw-v0-res.json @@ -0,0 +1 @@ +{"results": {"pile_freelaw": {"bits_per_byte": 3.16238943008513e-05, "byte_perplexity": 1.0000316243943415, "word_perplexity": 1.000203169094218}}, "versions": {"pile_freelaw": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_freelaw-v1-loglikelihood_rolling b/lm-evaluation-harness/tests/testdata/pile_freelaw-v1-loglikelihood_rolling new file mode 100644 index 0000000000000000000000000000000000000000..7b5771f4911f3069217d75d12cbdfa1a579b6663 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_freelaw-v1-loglikelihood_rolling @@ -0,0 +1 @@ +d77f3f68aadd6cbf1290c2f6737b2ed5d5c2a60e4c81a65c280f207783caabe1 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_freelaw-v1-res.json b/lm-evaluation-harness/tests/testdata/pile_freelaw-v1-res.json new file mode 100644 index 0000000000000000000000000000000000000000..dd0e0bac36b116bddbcd70d4327c3cdb3e3630e9 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_freelaw-v1-res.json @@ -0,0 +1 @@ +{"results": {"pile_freelaw": {"bits_per_byte": 4.5623635481434923e-05, "byte_perplexity": 1.0000316243943415, "word_perplexity": 1.000203169094218}}, "versions": {"pile_freelaw": 1}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_github-v0-loglikelihood_rolling b/lm-evaluation-harness/tests/testdata/pile_github-v0-loglikelihood_rolling new file mode 100644 index 0000000000000000000000000000000000000000..cf8251e4f68e2e893624142031e80d4d5777f4f2 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_github-v0-loglikelihood_rolling @@ -0,0 +1 @@ +df384c3df3d8f53273e97127c5bb84c17e638acad7d6bc9c91f6dee96d43b639 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_github-v0-res.json b/lm-evaluation-harness/tests/testdata/pile_github-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..bdabf399695d155026643eacca7954c5f87009d5 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_github-v0-res.json @@ -0,0 +1 @@ +{"results": {"pile_github": {"bits_per_byte": 9.540627613754646e-05, "byte_perplexity": 1.0000954108274611, "word_perplexity": 1.0009643183931227}}, "versions": {"pile_github": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_github-v1-loglikelihood_rolling b/lm-evaluation-harness/tests/testdata/pile_github-v1-loglikelihood_rolling new file mode 100644 index 0000000000000000000000000000000000000000..cf8251e4f68e2e893624142031e80d4d5777f4f2 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_github-v1-loglikelihood_rolling @@ -0,0 +1 @@ +df384c3df3d8f53273e97127c5bb84c17e638acad7d6bc9c91f6dee96d43b639 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_github-v1-res.json b/lm-evaluation-harness/tests/testdata/pile_github-v1-res.json new file mode 100644 index 0000000000000000000000000000000000000000..cc06a45501fb498db32e56d0677ef01f10869cc9 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_github-v1-res.json @@ -0,0 +1 @@ +{"results": {"pile_github": {"bits_per_byte": 0.00013764216145332133, "byte_perplexity": 1.0000954108274611, "word_perplexity": 1.0009643183931227}}, "versions": {"pile_github": 1}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_gutenberg-v0-loglikelihood_rolling b/lm-evaluation-harness/tests/testdata/pile_gutenberg-v0-loglikelihood_rolling new file mode 100644 index 0000000000000000000000000000000000000000..bd7b15927f717baab5b7ce2e9d659dda6d681769 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_gutenberg-v0-loglikelihood_rolling @@ -0,0 +1 @@ +02a559f74a9105145e7d4d9c5ddea372b5b4938f5368dc8ffafc39cbe3b4c7ef \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_gutenberg-v0-res.json b/lm-evaluation-harness/tests/testdata/pile_gutenberg-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..757ef06f79c938e6968583eb58a135fad23c897e --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_gutenberg-v0-res.json @@ -0,0 +1 @@ +{"results": {"pile_gutenberg": {"bits_per_byte": 1.2443606332351536e-06, "byte_perplexity": 1.0000012443614075, "word_perplexity": 1.0000072174665404}}, "versions": {"pile_gutenberg": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_gutenberg-v1-loglikelihood_rolling b/lm-evaluation-harness/tests/testdata/pile_gutenberg-v1-loglikelihood_rolling new file mode 100644 index 0000000000000000000000000000000000000000..bd7b15927f717baab5b7ce2e9d659dda6d681769 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_gutenberg-v1-loglikelihood_rolling @@ -0,0 +1 @@ +02a559f74a9105145e7d4d9c5ddea372b5b4938f5368dc8ffafc39cbe3b4c7ef \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_gutenberg-v1-res.json b/lm-evaluation-harness/tests/testdata/pile_gutenberg-v1-res.json new file mode 100644 index 0000000000000000000000000000000000000000..6d22ed3ff50eaa5a68f8a5ad1ac4d3828f74f81f --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_gutenberg-v1-res.json @@ -0,0 +1 @@ +{"results": {"pile_gutenberg": {"bits_per_byte": 1.7952329146458065e-06, "byte_perplexity": 1.0000012443614075, "word_perplexity": 1.0000072174665404}}, "versions": {"pile_gutenberg": 1}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_hackernews-v0-loglikelihood_rolling b/lm-evaluation-harness/tests/testdata/pile_hackernews-v0-loglikelihood_rolling new file mode 100644 index 0000000000000000000000000000000000000000..48b767bfe706bb035e4553ea9c4119347303bab9 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_hackernews-v0-loglikelihood_rolling @@ -0,0 +1 @@ +ec1082ee5a5326e0d57aa4e73b634937140c1de9af95f154e8ab57b05d9b422b \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_hackernews-v0-res.json b/lm-evaluation-harness/tests/testdata/pile_hackernews-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..68578fe4c952b8bccb26700be82df67450c558dd --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_hackernews-v0-res.json @@ -0,0 +1 @@ +{"results": {"pile_hackernews": {"bits_per_byte": 0.00010170276359193358, "byte_perplexity": 1.0001017079354932, "word_perplexity": 1.0006273924348839}}, "versions": {"pile_hackernews": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_hackernews-v1-loglikelihood_rolling b/lm-evaluation-harness/tests/testdata/pile_hackernews-v1-loglikelihood_rolling new file mode 100644 index 0000000000000000000000000000000000000000..48b767bfe706bb035e4553ea9c4119347303bab9 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_hackernews-v1-loglikelihood_rolling @@ -0,0 +1 @@ +ec1082ee5a5326e0d57aa4e73b634937140c1de9af95f154e8ab57b05d9b422b \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_hackernews-v1-res.json b/lm-evaluation-harness/tests/testdata/pile_hackernews-v1-res.json new file mode 100644 index 0000000000000000000000000000000000000000..ea135278b720703540187531afb0ef82e7d6a1ce --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_hackernews-v1-res.json @@ -0,0 +1 @@ +{"results": {"pile_hackernews": {"bits_per_byte": 0.00014672607267878518, "byte_perplexity": 1.0001017079354932, "word_perplexity": 1.0006273924348839}}, "versions": {"pile_hackernews": 1}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_nih-exporter-v0-loglikelihood_rolling b/lm-evaluation-harness/tests/testdata/pile_nih-exporter-v0-loglikelihood_rolling new file mode 100644 index 0000000000000000000000000000000000000000..5f76588a813eebe7f0958a07253480d30de2ccf3 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_nih-exporter-v0-loglikelihood_rolling @@ -0,0 +1 @@ +520ea6e04e8a39dc0b5f63a837429a78a40e63d39d109096101feb8c5b2cf8d8 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_nih-exporter-v0-res.json b/lm-evaluation-harness/tests/testdata/pile_nih-exporter-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..1c7bb56c6dc6cec7e2677317b3f9888293a65b92 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_nih-exporter-v0-res.json @@ -0,0 +1 @@ +{"results": {"pile_nih-exporter": {"bits_per_byte": 0.00024394433346975716, "byte_perplexity": 1.0002439740903082, "word_perplexity": 1.0016712202288802}}, "versions": {"pile_nih-exporter": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_nih-exporter-v1-loglikelihood_rolling b/lm-evaluation-harness/tests/testdata/pile_nih-exporter-v1-loglikelihood_rolling new file mode 100644 index 0000000000000000000000000000000000000000..5f76588a813eebe7f0958a07253480d30de2ccf3 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_nih-exporter-v1-loglikelihood_rolling @@ -0,0 +1 @@ +520ea6e04e8a39dc0b5f63a837429a78a40e63d39d109096101feb8c5b2cf8d8 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_nih-exporter-v1-res.json b/lm-evaluation-harness/tests/testdata/pile_nih-exporter-v1-res.json new file mode 100644 index 0000000000000000000000000000000000000000..0e40fc8268a77618471344585bc1a1586fd69e0f --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_nih-exporter-v1-res.json @@ -0,0 +1 @@ +{"results": {"pile_nih-exporter": {"bits_per_byte": 0.00035193728014978225, "byte_perplexity": 1.0002439740903082, "word_perplexity": 1.0016712202288802}}, "versions": {"pile_nih-exporter": 1}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_opensubtitles-v0-loglikelihood_rolling b/lm-evaluation-harness/tests/testdata/pile_opensubtitles-v0-loglikelihood_rolling new file mode 100644 index 0000000000000000000000000000000000000000..47805d3b5fe82555e4d61a90b43c157c974ddabc --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_opensubtitles-v0-loglikelihood_rolling @@ -0,0 +1 @@ +0f1c23a1f4ddec0c2b1ff34de8d1505b0eb9e2868d8edbcc1b6de13d02f32036 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_opensubtitles-v0-res.json b/lm-evaluation-harness/tests/testdata/pile_opensubtitles-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..f718e515ba0cedfa5156b3a260d50ed55efc32e4 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_opensubtitles-v0-res.json @@ -0,0 +1 @@ +{"results": {"pile_opensubtitles": {"bits_per_byte": 1.5213441136639177e-05, "byte_perplexity": 1.0000152135568616, "word_perplexity": 1.0000856162053249}}, "versions": {"pile_opensubtitles": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_opensubtitles-v1-loglikelihood_rolling b/lm-evaluation-harness/tests/testdata/pile_opensubtitles-v1-loglikelihood_rolling new file mode 100644 index 0000000000000000000000000000000000000000..47805d3b5fe82555e4d61a90b43c157c974ddabc --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_opensubtitles-v1-loglikelihood_rolling @@ -0,0 +1 @@ +0f1c23a1f4ddec0c2b1ff34de8d1505b0eb9e2868d8edbcc1b6de13d02f32036 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_opensubtitles-v1-res.json b/lm-evaluation-harness/tests/testdata/pile_opensubtitles-v1-res.json new file mode 100644 index 0000000000000000000000000000000000000000..1468294732b13576161fc3824a479028d5bdb0ba --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_opensubtitles-v1-res.json @@ -0,0 +1 @@ +{"results": {"pile_opensubtitles": {"bits_per_byte": 2.1948356082685497e-05, "byte_perplexity": 1.0000152135568616, "word_perplexity": 1.0000856162053249}}, "versions": {"pile_opensubtitles": 1}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_openwebtext2-v0-loglikelihood_rolling b/lm-evaluation-harness/tests/testdata/pile_openwebtext2-v0-loglikelihood_rolling new file mode 100644 index 0000000000000000000000000000000000000000..22046e440584d0df85ceeed057ad2c0633273782 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_openwebtext2-v0-loglikelihood_rolling @@ -0,0 +1 @@ +5d6c19665f429ab1ccbe027da67f42bdaf219f819ab093673976eee55e015ff4 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_openwebtext2-v0-res.json b/lm-evaluation-harness/tests/testdata/pile_openwebtext2-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..ead8d0b0bffe2da65e2602c7f2f352eeb404ef26 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_openwebtext2-v0-res.json @@ -0,0 +1 @@ +{"results": {"pile_openwebtext2": {"bits_per_byte": 0.00012809520662477846, "byte_perplexity": 1.000128103411166, "word_perplexity": 1.0007951516532847}}, "versions": {"pile_openwebtext2": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_openwebtext2-v1-loglikelihood_rolling b/lm-evaluation-harness/tests/testdata/pile_openwebtext2-v1-loglikelihood_rolling new file mode 100644 index 0000000000000000000000000000000000000000..22046e440584d0df85ceeed057ad2c0633273782 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_openwebtext2-v1-loglikelihood_rolling @@ -0,0 +1 @@ +5d6c19665f429ab1ccbe027da67f42bdaf219f819ab093673976eee55e015ff4 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_openwebtext2-v1-res.json b/lm-evaluation-harness/tests/testdata/pile_openwebtext2-v1-res.json new file mode 100644 index 0000000000000000000000000000000000000000..ca433e3c854780d034839c8e4d029cb6b5bfca1a --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_openwebtext2-v1-res.json @@ -0,0 +1 @@ +{"results": {"pile_openwebtext2": {"bits_per_byte": 0.000184802319359215, "byte_perplexity": 1.000128103411166, "word_perplexity": 1.0007951516532847}}, "versions": {"pile_openwebtext2": 1}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_philpapers-v0-loglikelihood_rolling b/lm-evaluation-harness/tests/testdata/pile_philpapers-v0-loglikelihood_rolling new file mode 100644 index 0000000000000000000000000000000000000000..4fbbc241ba9487c2513cdf46dbb76e004e401418 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_philpapers-v0-loglikelihood_rolling @@ -0,0 +1 @@ +339ba5d8c044c4a3ff9b9a8eaa24da1d6c01b72972074eb671a7da049eeb7047 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_philpapers-v0-res.json b/lm-evaluation-harness/tests/testdata/pile_philpapers-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..be561fe2f8a6fe5eba08c4c1efd113075da42e1f --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_philpapers-v0-res.json @@ -0,0 +1 @@ +{"results": {"pile_philpapers": {"bits_per_byte": 6.241575895982095e-06, "byte_perplexity": 1.0000062415953748, "word_perplexity": 1.0000409888564146}}, "versions": {"pile_philpapers": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_philpapers-v1-loglikelihood_rolling b/lm-evaluation-harness/tests/testdata/pile_philpapers-v1-loglikelihood_rolling new file mode 100644 index 0000000000000000000000000000000000000000..4fbbc241ba9487c2513cdf46dbb76e004e401418 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_philpapers-v1-loglikelihood_rolling @@ -0,0 +1 @@ +339ba5d8c044c4a3ff9b9a8eaa24da1d6c01b72972074eb671a7da049eeb7047 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_philpapers-v1-res.json b/lm-evaluation-harness/tests/testdata/pile_philpapers-v1-res.json new file mode 100644 index 0000000000000000000000000000000000000000..5a2f77678abc264edf433a4eb98da08fc20b1dfc --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_philpapers-v1-res.json @@ -0,0 +1 @@ +{"results": {"pile_philpapers": {"bits_per_byte": 9.004690592465457e-06, "byte_perplexity": 1.0000062415953748, "word_perplexity": 1.0000409888564146}}, "versions": {"pile_philpapers": 1}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/pile_pile-cc-v0-res.json b/lm-evaluation-harness/tests/testdata/pile_pile-cc-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..383233f259507dfdbc6556834185f1eb6161f9cd --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/pile_pile-cc-v0-res.json @@ -0,0 +1 @@ +{"results": {"pile_pile-cc": {"bits_per_byte": 0.00011234131907228174, "byte_perplexity": 1.0001123476295946, "word_perplexity": 1.0006738958554477}}, "versions": {"pile_pile-cc": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/webqs-v0-res.json b/lm-evaluation-harness/tests/testdata/webqs-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..9f0fdc76cab096c80a87295773054510803ba218 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/webqs-v0-res.json @@ -0,0 +1 @@ +{"results": {"webqs": {"acc": 0.0, "acc_stderr": 0.0}}, "versions": {"webqs": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/wic-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/wic-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..d27430a9a2eab0a6a5e265e249237201a4a56061 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/wic-v0-loglikelihood @@ -0,0 +1 @@ +403a08da05e4c44d7e3dd3358382a7ba489c41d223e24cd1a9ed82ef1a2d004b \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/wic-v0-res.json b/lm-evaluation-harness/tests/testdata/wic-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..eadc573ed3d6a9b8b9bd924896ef5d719a53d5d1 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/wic-v0-res.json @@ -0,0 +1 @@ +{"results": {"wic": {"acc": 0.49216300940438873, "acc_stderr": 0.01980828765781383}}, "versions": {"wic": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/wikitext-v0-loglikelihood_rolling b/lm-evaluation-harness/tests/testdata/wikitext-v0-loglikelihood_rolling new file mode 100644 index 0000000000000000000000000000000000000000..f09af45a38c0de097358c587420858c7a53a10aa --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/wikitext-v0-loglikelihood_rolling @@ -0,0 +1 @@ +b6f83e6cf7535ee41b0057c3e2ec2cf7f2fa5a9119b305c479a83091d1142b2c \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/wikitext-v0-res.json b/lm-evaluation-harness/tests/testdata/wikitext-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..9ac0c37bb5aa8cdde37bf84c61a0d020c8a03900 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/wikitext-v0-res.json @@ -0,0 +1 @@ +{"results": {"wikitext": {"bits_per_byte": 2.219817611605802e-05, "byte_perplexity": 1.0000221984224973, "word_perplexity": 1.000118710696617}}, "versions": {"wikitext": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/wikitext-v1-loglikelihood_rolling b/lm-evaluation-harness/tests/testdata/wikitext-v1-loglikelihood_rolling new file mode 100644 index 0000000000000000000000000000000000000000..f09af45a38c0de097358c587420858c7a53a10aa --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/wikitext-v1-loglikelihood_rolling @@ -0,0 +1 @@ +b6f83e6cf7535ee41b0057c3e2ec2cf7f2fa5a9119b305c479a83091d1142b2c \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/wikitext-v1-res.json b/lm-evaluation-harness/tests/testdata/wikitext-v1-res.json new file mode 100644 index 0000000000000000000000000000000000000000..122098aec22e39599f1d3bffbb4bf619131d2335 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/wikitext-v1-res.json @@ -0,0 +1 @@ +{"results": {"wikitext": {"bits_per_byte": 3.202519859941674e-05, "byte_perplexity": 1.0000221984224973, "word_perplexity": 1.000118710696617}}, "versions": {"wikitext": 1}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/wikitext_10_hf_pretrained-EleutherAI-pythia-14m-dtype-float32-device-cpu.txt b/lm-evaluation-harness/tests/testdata/wikitext_10_hf_pretrained-EleutherAI-pythia-14m-dtype-float32-device-cpu.txt new file mode 100644 index 0000000000000000000000000000000000000000..654e63ee7cb083d729d9be56e0d9d71ef1805928 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/wikitext_10_hf_pretrained-EleutherAI-pythia-14m-dtype-float32-device-cpu.txt @@ -0,0 +1,5 @@ +| Tasks |Version|Filter|n-shot| Metric | | Value | |Stderr| +|--------|------:|------|-----:|---------------|---|-------:|---|------| +|wikitext| 2|none | 0|bits_per_byte |↓ | 1.3394|± | N/A| +| | |none | 0|byte_perplexity|↓ | 2.5304|± | N/A| +| | |none | 0|word_perplexity|↓ |130.4801|± | N/A| \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/winogrande-v0-loglikelihood b/lm-evaluation-harness/tests/testdata/winogrande-v0-loglikelihood new file mode 100644 index 0000000000000000000000000000000000000000..97866f6ce45cb9a213d27310a78b7cdeab23bc9a --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/winogrande-v0-loglikelihood @@ -0,0 +1 @@ +90a3eff49de9173964d46f5ed57bcf9a78a72dd1bfe0e5323b25cebb40b49ea9 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/winogrande-v0-res.json b/lm-evaluation-harness/tests/testdata/winogrande-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..9fa7903a56d2fb48abcd215bb587bc69c00f4aa6 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/winogrande-v0-res.json @@ -0,0 +1 @@ +{"results": {"winogrande": {"acc": 0.516179952644041, "acc_stderr": 0.014045126130978606}}, "versions": {"winogrande": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/wmt14-en-fr-v0-greedy_until b/lm-evaluation-harness/tests/testdata/wmt14-en-fr-v0-greedy_until new file mode 100644 index 0000000000000000000000000000000000000000..6d48d5579e95eb72bdc6c4dc8b4149e5f495b55e --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/wmt14-en-fr-v0-greedy_until @@ -0,0 +1 @@ +368ae7eec0f902b5123f2d5197caa5109a23942011c53fe68d9eaeee20180e46 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/wmt14-en-fr-v0-res.json b/lm-evaluation-harness/tests/testdata/wmt14-en-fr-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..1aa13f02854c8eec0591be980486afe48d7f97a9 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/wmt14-en-fr-v0-res.json @@ -0,0 +1 @@ +{"results": {"wmt14-en-fr": {"bleu": 0.0, "bleu_stderr": 0.0, "chrf": 0.011284118461117099, "chrf_stderr": 7.340651275964445e-05, "ter": 1.0, "ter_stderr": 0.0}}, "versions": {"wmt14-en-fr": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/wmt14-fr-en-v0-greedy_until b/lm-evaluation-harness/tests/testdata/wmt14-fr-en-v0-greedy_until new file mode 100644 index 0000000000000000000000000000000000000000..7249d39990f9aea60634b07c975f735983bade89 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/wmt14-fr-en-v0-greedy_until @@ -0,0 +1 @@ +c1d9f7283755fbdd7ecd6cc4278b0ac25a80ac256b7071ea5f839ccd038e5974 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/wmt14-fr-en-v0-res.json b/lm-evaluation-harness/tests/testdata/wmt14-fr-en-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..5261876f55a69dcaf33b3842690f81c12eb42f3a --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/wmt14-fr-en-v0-res.json @@ -0,0 +1 @@ +{"results": {"wmt14-fr-en": {"bleu": 0.0, "bleu_stderr": 0.0, "chrf": 0.01275083169440515, "chrf_stderr": 8.45474998563806e-05, "ter": 1.0, "ter_stderr": 0.0}}, "versions": {"wmt14-fr-en": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/wmt16-de-en-v0-greedy_until b/lm-evaluation-harness/tests/testdata/wmt16-de-en-v0-greedy_until new file mode 100644 index 0000000000000000000000000000000000000000..75f1072b6e7f2bdd9ecd98987c86fefd3375fb6d --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/wmt16-de-en-v0-greedy_until @@ -0,0 +1 @@ +d30e23e38d9a45b9c31e1dfd14b58d0b7020df4b9c8a1c697aa6bc5fba8ce08a \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/wmt16-de-en-v0-res.json b/lm-evaluation-harness/tests/testdata/wmt16-de-en-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..826e0382abb32da67ac0dd0b271527b40fd3b6ae --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/wmt16-de-en-v0-res.json @@ -0,0 +1 @@ +{"results": {"wmt16-de-en": {"bleu": 0.0, "bleu_stderr": 0.0, "chrf": 0.013700416764482968, "chrf_stderr": 0.00016071651360909355, "ter": 1.0, "ter_stderr": 0.0}}, "versions": {"wmt16-de-en": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/wmt16-en-de-v0-greedy_until b/lm-evaluation-harness/tests/testdata/wmt16-en-de-v0-greedy_until new file mode 100644 index 0000000000000000000000000000000000000000..45eaaaca8c5892944b1b9c9af0c469e3c63e4881 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/wmt16-en-de-v0-greedy_until @@ -0,0 +1 @@ +d71e2074af3770e9b29ac561caf2e1c29ad6b0dc50ec2e7bcc5501747b11f0da \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/wmt16-en-de-v0-res.json b/lm-evaluation-harness/tests/testdata/wmt16-en-de-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..88bee7ffa69b1bf7accdd56a3870f61d4c0453da --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/wmt16-en-de-v0-res.json @@ -0,0 +1 @@ +{"results": {"wmt16-en-de": {"bleu": 0.0, "bleu_stderr": 0.0, "chrf": 0.010909486120840577, "chrf_stderr": 0.000122611124711072, "ter": 1.0, "ter_stderr": 0.0}}, "versions": {"wmt16-en-de": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/wmt16-en-ro-v0-greedy_until b/lm-evaluation-harness/tests/testdata/wmt16-en-ro-v0-greedy_until new file mode 100644 index 0000000000000000000000000000000000000000..291492556e5182600291565c640a463da7f00616 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/wmt16-en-ro-v0-greedy_until @@ -0,0 +1 @@ +4be7fdda313394f19b5995b00ada1dfa3bb158ee1f020ef8d07ecea260fa60b2 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/wmt16-en-ro-v0-res.json b/lm-evaluation-harness/tests/testdata/wmt16-en-ro-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..babb8d2d74fb5585cf9578f8b1dc8be3dde43f63 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/wmt16-en-ro-v0-res.json @@ -0,0 +1 @@ +{"results": {"wmt16-en-ro": {"bleu": 0.0, "bleu_stderr": 0.0, "chrf": 0.012004814364156886, "chrf_stderr": 6.424423961332661e-05, "ter": 1.0, "ter_stderr": 0.0}}, "versions": {"wmt16-en-ro": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/wmt16-ro-en-v0-greedy_until b/lm-evaluation-harness/tests/testdata/wmt16-ro-en-v0-greedy_until new file mode 100644 index 0000000000000000000000000000000000000000..fbcac1b7e3887c6ffa8fd6da6e21595fb0c49a4f --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/wmt16-ro-en-v0-greedy_until @@ -0,0 +1 @@ +d1b7c50751b0d5d7470b7f49f2bab9d09792c91460fc92cc34f06617013d7c65 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/wmt16-ro-en-v0-res.json b/lm-evaluation-harness/tests/testdata/wmt16-ro-en-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..267763793d5fa5a16c41cbcdd9eb7b134cd34cea --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/wmt16-ro-en-v0-res.json @@ -0,0 +1 @@ +{"results": {"wmt16-ro-en": {"bleu": 0.0, "bleu_stderr": 0.0, "chrf": 0.01262029828861831, "chrf_stderr": 0.00014507496111350828, "ter": 1.0, "ter_stderr": 0.0}}, "versions": {"wmt16-ro-en": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/wmt20-cs-en-v0-greedy_until b/lm-evaluation-harness/tests/testdata/wmt20-cs-en-v0-greedy_until new file mode 100644 index 0000000000000000000000000000000000000000..7bcf240b7090e406259d4bfc090d1eb22ec6e291 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/wmt20-cs-en-v0-greedy_until @@ -0,0 +1 @@ +bfead9efdb1b2402a414c55929c8d8f956585f938a35466931d44e81d89cfe00 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/wmt20-de-en-v0-greedy_until b/lm-evaluation-harness/tests/testdata/wmt20-de-en-v0-greedy_until new file mode 100644 index 0000000000000000000000000000000000000000..c02fb9875d5354fdb0892b7493a822ee4af9d6c2 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/wmt20-de-en-v0-greedy_until @@ -0,0 +1 @@ +d13b5a6915ca86ac6c6ebc50d9be0d0be3dfca600c12e896df53190d875de74d \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/wmt20-de-en-v0-res.json b/lm-evaluation-harness/tests/testdata/wmt20-de-en-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..790424fe4f226224642530ba7fd53a59eec4caa0 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/wmt20-de-en-v0-res.json @@ -0,0 +1 @@ +{"results": {"wmt20-de-en": {"bleu": 0.0, "bleu_stderr": 0.0, "chrf": 0.006703243310670055, "chrf_stderr": 0.0001292711927988445, "ter": 1.0, "ter_stderr": 0.0}}, "versions": {"wmt20-de-en": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/wmt20-de-fr-v0-greedy_until b/lm-evaluation-harness/tests/testdata/wmt20-de-fr-v0-greedy_until new file mode 100644 index 0000000000000000000000000000000000000000..7cb9424082836f0d56afe809cf44c78fc844d993 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/wmt20-de-fr-v0-greedy_until @@ -0,0 +1 @@ +7f197bc281d6dbf9425900ef0dee7175021c43e355050f149f43b161c52bf0b0 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/wmt20-de-fr-v0-res.json b/lm-evaluation-harness/tests/testdata/wmt20-de-fr-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..79a0d12fe6f5750749e56dc3919283f71d021fa0 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/wmt20-de-fr-v0-res.json @@ -0,0 +1 @@ +{"results": {"wmt20-de-fr": {"bleu": 0.0, "bleu_stderr": 0.0, "chrf": 0.011897164096796364, "chrf_stderr": 0.00010158164726118333, "ter": 1.0, "ter_stderr": 0.0}}, "versions": {"wmt20-de-fr": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/wmt20-en-cs-v0-greedy_until b/lm-evaluation-harness/tests/testdata/wmt20-en-cs-v0-greedy_until new file mode 100644 index 0000000000000000000000000000000000000000..d14fc4939aecb7bb40458c34954c1242d9f20501 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/wmt20-en-cs-v0-greedy_until @@ -0,0 +1 @@ +5a34e6863bf6965afd31653de50bac5fecf58db65dbaba46921504a2b7463786 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/wmt20-en-cs-v0-res.json b/lm-evaluation-harness/tests/testdata/wmt20-en-cs-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..2ba9db70d3579ff23ee70c3b16eb92d7d87144e6 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/wmt20-en-cs-v0-res.json @@ -0,0 +1 @@ +{"results": {"wmt20-en-cs": {"bleu": 0.0, "bleu_stderr": 0.0, "chrf": 0.009879653442394573, "chrf_stderr": 8.210293331159994e-05, "ter": 1.0, "ter_stderr": 0.0}}, "versions": {"wmt20-en-cs": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/wmt20-en-de-v0-greedy_until b/lm-evaluation-harness/tests/testdata/wmt20-en-de-v0-greedy_until new file mode 100644 index 0000000000000000000000000000000000000000..c4078efd996d010eac102fe23de50fdbbe0310d9 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/wmt20-en-de-v0-greedy_until @@ -0,0 +1 @@ +b6e9c305766ea23ce1027309f83c6d4c2ce8948d70b63a7858586ca34050d7fb \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/wmt20-en-de-v0-res.json b/lm-evaluation-harness/tests/testdata/wmt20-en-de-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..183e66270a61dcb463076306b2768dde7995162f --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/wmt20-en-de-v0-res.json @@ -0,0 +1 @@ +{"results": {"wmt20-en-de": {"bleu": 0.0, "bleu_stderr": 0.0, "chrf": 0.007148103038872972, "chrf_stderr": 9.594096858911254e-05, "ter": 1.0, "ter_stderr": 0.0}}, "versions": {"wmt20-en-de": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/wmt20-en-iu-v0-greedy_until b/lm-evaluation-harness/tests/testdata/wmt20-en-iu-v0-greedy_until new file mode 100644 index 0000000000000000000000000000000000000000..d26bb4f92a03612cf3a4170733973e39870164b7 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/wmt20-en-iu-v0-greedy_until @@ -0,0 +1 @@ +f5688199890a48f73f2cc04a2152e35190f0e0ddd40e629fa24ee39d423ea389 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/wmt20-en-iu-v0-res.json b/lm-evaluation-harness/tests/testdata/wmt20-en-iu-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..22f042eb4eba6e6e662e46807232679782f7b6b9 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/wmt20-en-iu-v0-res.json @@ -0,0 +1 @@ +{"results": {"wmt20-en-iu": {"bleu": 0.0, "bleu_stderr": 0.0, "chrf": 0.00011803644548940443, "chrf_stderr": 2.175287038623409e-05, "ter": 1.0, "ter_stderr": 0.0}}, "versions": {"wmt20-en-iu": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/wmt20-en-ja-v0-greedy_until b/lm-evaluation-harness/tests/testdata/wmt20-en-ja-v0-greedy_until new file mode 100644 index 0000000000000000000000000000000000000000..9777002c79830918a3939ec6978d606ae967ffe6 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/wmt20-en-ja-v0-greedy_until @@ -0,0 +1 @@ +7fe61f5847a51e93e97c84b39f4420978727754e4b6cf636a27851c615857530 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/wmt20-en-ja-v0-res.json b/lm-evaluation-harness/tests/testdata/wmt20-en-ja-v0-res.json new file mode 100644 index 0000000000000000000000000000000000000000..57bad300d7f05b7eb0908ad655b0fc90a2bb9c26 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/wmt20-en-ja-v0-res.json @@ -0,0 +1 @@ +{"results": {"wmt20-en-ja": {"bleu": 0.0, "bleu_stderr": 0.0, "chrf": 4.1308658294778584e-05, "chrf_stderr": 2.0456539027807417e-05, "ter": 1.0, "ter_stderr": 0.0}}, "versions": {"wmt20-en-ja": 0}} \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/wmt20-en-ja-v1-greedy_until b/lm-evaluation-harness/tests/testdata/wmt20-en-ja-v1-greedy_until new file mode 100644 index 0000000000000000000000000000000000000000..9777002c79830918a3939ec6978d606ae967ffe6 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/wmt20-en-ja-v1-greedy_until @@ -0,0 +1 @@ +7fe61f5847a51e93e97c84b39f4420978727754e4b6cf636a27851c615857530 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/wmt20-en-km-v0-greedy_until b/lm-evaluation-harness/tests/testdata/wmt20-en-km-v0-greedy_until new file mode 100644 index 0000000000000000000000000000000000000000..ddce46a79fdcb08c3eee1a534c11fc4dd796be53 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/wmt20-en-km-v0-greedy_until @@ -0,0 +1 @@ +eb5365c46f22ffec9a157991627d6e1fd1117fccffaedfc73619e93bafb5a408 \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testdata/wmt20-ta-en-v0-greedy_until b/lm-evaluation-harness/tests/testdata/wmt20-ta-en-v0-greedy_until new file mode 100644 index 0000000000000000000000000000000000000000..f0f65972451ff666399f7b2c81194c4b892ac783 --- /dev/null +++ b/lm-evaluation-harness/tests/testdata/wmt20-ta-en-v0-greedy_until @@ -0,0 +1 @@ +111ea3efdc08f1cf536631b9426c3a20e482c575d009d2a8c71f59c027578eec \ No newline at end of file diff --git a/lm-evaluation-harness/tests/testyamls/test-01.yaml b/lm-evaluation-harness/tests/testyamls/test-01.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4e4367eeb3651bd49540009949fc080b9dba0a59 --- /dev/null +++ b/lm-evaluation-harness/tests/testyamls/test-01.yaml @@ -0,0 +1,45 @@ +group: test-1 +group_alias: test 1 +task: + - piqa # string task + - ai2_arc # string tag + - task: super-glue-lm-eval-v1 # Should this be spread out? + num_fewshot: 3 + - task: swag # dict registered task + num_fewshot: 2 + - task: mmlu + num_fewshot: 5 + - group: nli-tasks # dict group + task: + - anli + - boolq + - sglue_rte + num_fewshot: 4 + metric_list: + - metric: brier_score + - task: sciq # dict registered task duplicate + task_alias: sciq 2-shot + num_fewshot: 2 + - task: sciq # dict registered task duplicate + task_alias: sciq 4-shot + num_fewshot: 4 + - task: sciq # dict registered task duplicate + task_alias: sciq 6-shot + num_fewshot: 6 + - task: siqa_custom # dict task + dataset_path: social_i_qa + dataset_name: null + output_type: multiple_choice + training_split: train + validation_split: validation + doc_to_text: "Question: {{context}} {{question}}\nAnswer:" + target_delimiter: " " + doc_to_choice: + - "{{answerA}}" + - "{{answerB}}" + - "{{answerC}}" + doc_to_target: "{{ (label|int) - 1 }}" + metric_list: + - metric: acc + aggregation: mean + higher_is_better: true diff --git a/lm-evaluation-harness/tests/utils.py b/lm-evaluation-harness/tests/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..0c0398029a58f968bdb1c89d6faee3c9a52144b4 --- /dev/null +++ b/lm-evaluation-harness/tests/utils.py @@ -0,0 +1,48 @@ +import os +from typing import List, Union + +from lm_eval.utils import load_yaml_config + + +# {{{CI}}} +# This is the path where the output for the changed files for the tasks folder is stored +# FILE_PATH = file_path = ".github/outputs/tasks_all_changed_and_modified_files.txt" + + +# reads a text file and returns a list of words +# used to read the output of the changed txt from tj-actions/changed-files +def load_changed_files(file_path: str) -> List[str]: + with open(file_path, "r", encoding="utf-8") as f: + content = f.read() + words_list = list(content.split()) + return words_list + + +# checks the txt file for list of changed files. +# if file ends with .yaml then check yaml and load the config. +# if the config task is a string, it's a task config. +# if the config task is a list, it's a group config. +def parser(full_path: List[str]) -> List[str]: + _output = set() + for x in full_path: + if x.endswith(".yaml") and os.path.exists(x): + config = load_yaml_config(x, mode="simple") + if isinstance(config["task"], str): + _output.add(config["task"]) + elif isinstance(config["task"], list): + _output.add(config["group"]) + return list(_output) + + +def new_tasks() -> Union[List[str], None]: + FILENAME = ".github/outputs/tasks_all_changed_and_modified_files.txt" + if os.path.exists(FILENAME): + # If tasks folder has changed then we get the list of files from FILENAME + # and parse the yaml files to get the task names. + return parser(load_changed_files(FILENAME)) + if os.getenv("API") is not None: + # Or if API has changed then we set the ENV variable API to True + # and run given tasks. + return ["arc_easy", "hellaswag", "piqa", "wikitext"] + # if both not true just do arc_easy + return None diff --git a/lm-quant-toolkit/.coveragerc b/lm-quant-toolkit/.coveragerc new file mode 100644 index 0000000000000000000000000000000000000000..7bd0571638c2961700b723fee6a004f1e8a63faf --- /dev/null +++ b/lm-quant-toolkit/.coveragerc @@ -0,0 +1,3 @@ +[run] +omit = + src/llm_quant_eval/tests/* diff --git a/lm-quant-toolkit/.deps/AutoGPTQ/.gitignore b/lm-quant-toolkit/.deps/AutoGPTQ/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..68bc17f9ff2104a9d7b6777058bb4c343ca72609 --- /dev/null +++ b/lm-quant-toolkit/.deps/AutoGPTQ/.gitignore @@ -0,0 +1,160 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ diff --git a/lm-quant-toolkit/.editorconfig b/lm-quant-toolkit/.editorconfig new file mode 100644 index 0000000000000000000000000000000000000000..a0150356a1e6fb915bb4cdd61be1fe41bd0e1d66 --- /dev/null +++ b/lm-quant-toolkit/.editorconfig @@ -0,0 +1,29 @@ +# EditorConfig: http://editorconfig.org/ + +root = true + +[*] +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true +indent_style = space + +[*.py] +max_line_length = 80 +indent_style = space +indent_size = 4 + +[*.md] +max_line_length = 80 + +[*.json] +indent_size = 2 + +[*.py] +indent_size = 4 + +[*.sh] +indent_size = 4 + +[*.{yml,yaml}] +indent_size = 2 diff --git a/lm-quant-toolkit/.gitignore b/lm-quant-toolkit/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..b198d631f304debca7f6624164cbedd6b11f9712 --- /dev/null +++ b/lm-quant-toolkit/.gitignore @@ -0,0 +1,26 @@ +.DS_Store +credential.ini +*-debug.py +*.log +*.bak +*.swo +*.swp +*.egg +*.egg/ +*.egg-info/ +*.pyc +.tox/ +_build/ +build +dist/ +.coverage +.cache +htmlcov +*requirement*.txt +*.pickle +*.parquet +.*venv/ +# R related files +.Rproj.user +.Rhistory +Rplots.pdf diff --git a/lm-quant-toolkit/LICENSE b/lm-quant-toolkit/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..ad396d05628ddd35d92180c0a834c538f18a1e83 --- /dev/null +++ b/lm-quant-toolkit/LICENSE @@ -0,0 +1,17 @@ +Copyright 2024 Justin Zhang + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the +“Software”), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS +OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/lm-quant-toolkit/MANIFEST.in b/lm-quant-toolkit/MANIFEST.in new file mode 100644 index 0000000000000000000000000000000000000000..533aa7df34b86451bca0a6d36817987b63462b3a --- /dev/null +++ b/lm-quant-toolkit/MANIFEST.in @@ -0,0 +1,5 @@ +include MANIFEST.in +include README.rst +include LICENSE +recursive-include docs *.rst conf.py Makefile +recursive-include src *.py diff --git a/lm-quant-toolkit/pyproject.toml b/lm-quant-toolkit/pyproject.toml new file mode 100644 index 0000000000000000000000000000000000000000..f4a6ea95ecf3b0ba338dd93fc425128425804e5c --- /dev/null +++ b/lm-quant-toolkit/pyproject.toml @@ -0,0 +1,70 @@ +[build-system] +requires = ['setuptools', 'setuptools_scm'] +build-backend = 'setuptools.build_meta' + +[project] +name = 'lm_quant_toolkit' +description = 'LLM Quantization Evaluation Harness' +readme = 'README.md' +license = {file = 'LICENSE'} +authors = [{name='Justin Zhang', email='schnell18@gmail.com'}] +dependencies = [ + # "-e git+https://github.com/casper-hansen/AutoAWQ@5f3785dcaa107ca76f5fa5355f459370c86f82d6#egg=autoawq", + # "-e git+ssh://git@github.com/schnell18/hqq.git@5ba2243049f0b96fe5d8843693f919faeb560c02#egg=hqq", + # "-e git+https://github.com/EleutherAI/lm-evaluation-harness@928e8bb6f50d1e93ef5d0bcaa81f8c5fd9a6f4d8#egg=lm_eval", + "accelerate>=0.30.1", + "bitsandbytes>0.37.0", + "antlr4-python3-runtime==4.11.0", + "datasets==2.20.0", + "Jinja2==3.1.4", + "langdetect==1.0.9", + "nltk==3.9.1", + "numpy==1.26.4", + "optimum>=1.21.4", + "pandas==2.2.2", + "safetensors==0.4.3", + "scikit-learn==1.4.2", + "scipy==1.13.0", + "sentencepiece==0.2.0", + "tokenizers>=0.19.1", + "torch>=2.1.0", + "tqdm==4.66.4", + "transformers>=4.41.2" + # "triton>=2.3.0", +] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT No Attribution License (MIT-0)", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: Implementation :: CPython", +] +requires-python = '>=3.7' +dynamic = ['version'] + +[project.scripts] +#fnorm = 'lm_quant_toolkit.prep.fnorm:main' + +[tools.setuptools] + package-dir = {'' = 'src'} + +[tools.setuptools.dynamic] +version = {attr = 'lm_quant_toolkit.version.version'} + +[tool.setuptools_scm] +version_scheme = 'python-simplified-semver' +local_scheme = 'no-local-version' + +[project.urls] +'Homepage' = 'https://github.com/schnell18/lm-quant-toolkit' + +[project.optional-dependencies] +dev = ["bumpver", "pip-tools"] +test = [ 'tox' ] +doc = ['sphinx']