Remove legacy fusion artifacts

benchmark_fusion_arithmark.py

@@ -1,290 +0,0 @@
-"""Score a fusion GPT checkpoint on ArithMark 2.0."""
-
-from __future__ import annotations
-
-import argparse
-from collections import Counter
-from contextlib import nullcontext
-import json
-from pathlib import Path
-import re
-import urllib.request
-
-import torch
-import torch.nn.functional as F
-from transformers import AutoModelForCausalLM, AutoTokenizer
-
-
-DATA_URL = (
-    "https://huggingface.co/datasets/AxiomicLabs/Arithmark-2.0/"
-    "resolve/main/arithmark_2.0.jsonl"
-)
-
-
-def ensure_data(path: Path) -> Path:
-    if path.exists():
-        return path
-    path.parent.mkdir(parents=True, exist_ok=True)
-    urllib.request.urlretrieve(DATA_URL, path)
-    return path
-
-
-def load_examples(path: Path, *, max_examples: int = 0) -> list[dict]:
-    examples = []
-    with path.open("r", encoding="utf-8") as handle:
-        for line in handle:
-            if not line.strip():
-                continue
-            examples.append(json.loads(line))
-            if max_examples > 0 and len(examples) >= max_examples:
-                break
-    return examples
-
-
-def _encoded_choice(
-    tokenizer,
-    context: str,
-    ending: str,
-) -> tuple[list[int], int]:
-    context_ids = tokenizer(context, add_special_tokens=False).input_ids
-    full_ids = tokenizer(context + ending, add_special_tokens=False).input_ids
-    continuation_length = len(full_ids) - len(context_ids)
-    return full_ids, continuation_length
-
-
-@torch.inference_mode()
-def evaluate(
-    model,
-    tokenizer,
-    examples: list[dict],
-    *,
-    device: torch.device,
-    batch_size: int,
-    dump_failures: bool = False,
-    failure_operator_count: int | None = None,
-    max_failures: int = 100,
-) -> dict:
-    correct = 0
-    total = 0
-    by_operator_count: dict[str, list[int]] = {}
-    by_topic: dict[str, list[int]] = {}
-    failures: list[dict] = []
-    failure_summary: Counter[tuple[str, str, str]] = Counter()
-    model.eval()
-    pad_id = tokenizer.pad_token_id
-    if pad_id is None:
-        pad_id = tokenizer.eos_token_id if tokenizer.eos_token_id is not None else 0
-
-    for start in range(0, len(examples), batch_size):
-        batch_examples = examples[start : start + batch_size]
-        encoded = []
-        offsets = []
-        for example in batch_examples:
-            flat_start = len(encoded)
-            encoded.extend(
-                _encoded_choice(tokenizer, example["ctx"], ending)
-                for ending in example["endings"]
-            )
-            offsets.append((flat_start, len(example["endings"])))
-
-        max_length = max(len(item[0]) for item in encoded)
-        input_ids = torch.full(
-            (len(encoded), max_length),
-            int(pad_id),
-            dtype=torch.long,
-            device=device,
-        )
-        attention_mask = torch.zeros_like(input_ids, dtype=torch.bool)
-        lengths = []
-        continuation_lengths = []
-        for row, (ids, continuation_length) in enumerate(encoded):
-            length = len(ids)
-            input_ids[row, :length] = torch.tensor(ids, device=device)
-            attention_mask[row, :length] = True
-            lengths.append(length)
-            continuation_lengths.append(continuation_length)
-
-        autocast = (
-            torch.autocast(device_type="cuda", dtype=torch.bfloat16)
-            if device.type == "cuda"
-            else nullcontext()
-        )
-        with autocast:
-            logits = model(
-                input_ids=input_ids,
-                attention_mask=attention_mask,
-            ).logits
-        log_probs = F.log_softmax(logits.float(), dim=-1)
-
-        for example_index, example in enumerate(batch_examples):
-            flat_start, choice_count = offsets[example_index]
-            likelihoods = []
-            for choice_index in range(choice_count):
-                row = flat_start + choice_index
-                length = lengths[row]
-                continuation_length = continuation_lengths[row]
-                continuation_start = length - continuation_length
-                likelihood = 0.0
-                for position in range(continuation_start, length):
-                    likelihood += float(
-                        log_probs[row, position - 1, input_ids[row, position]].item()
-                    )
-                likelihoods.append(likelihood)
-
-            prediction = max(range(choice_count), key=likelihoods.__getitem__)
-            label = int(example["label"])
-            matched = prediction == label
-            correct += int(matched)
-            total += 1
-            metadata = example.get("metadata", {})
-            operator_count = str(metadata.get("operator_count", "unknown"))
-            topic = str(metadata.get("topic", "unknown"))
-            for grouped, key in (
-                (by_operator_count, operator_count),
-                (by_topic, topic),
-            ):
-                group = grouped.setdefault(key, [0, 0])
-                group[0] += int(matched)
-                group[1] += 1
-
-            if not matched and dump_failures:
-                op_count_int = None
-                try:
-                    op_count_int = int(operator_count)
-                except ValueError:
-                    pass
-                if failure_operator_count is None or op_count_int == failure_operator_count:
-                    context = str(example["ctx"]).strip()
-                    expression = context[:-1].strip() if context.endswith("=") else context
-                    operands = [int(value) for value in re.findall(r"\d+", expression)]
-                    operator = "".join(re.findall(r"[+\-*/]", expression))
-                    predicted_answer = str(example["endings"][prediction]).strip()
-                    correct_answer = str(example["endings"][label]).strip()
-                    width = max((len(str(value)) for value in operands), default=0)
-                    failure_summary[(topic, operator, f"width={width}")] += 1
-                    if len(failures) < max_failures:
-                        failures.append(
-                            {
-                                "ctx": context,
-                                "topic": topic,
-                                "operator_count": operator_count,
-                                "operator": operator,
-                                "operands": operands,
-                                "max_operand_digits": width,
-                                "correct_answer": correct_answer,
-                                "predicted_answer": predicted_answer,
-                                "choices": [str(value).strip() for value in example["endings"]],
-                                "choice_scores": [round(value, 4) for value in likelihoods],
-                                "score_margin_correct_minus_predicted": round(
-                                    likelihoods[label] - likelihoods[prediction],
-                                    4,
-                                ),
-                            }
-                        )
-
-    results = {
-        "benchmark": "arithmark_2.0",
-        "model_type": "fusion_gpt",
-        "accuracy": correct / max(total, 1),
-        "correct": correct,
-        "total": total,
-        "by_operator_count": {
-            key: {
-                "accuracy": values[0] / max(values[1], 1),
-                "correct": values[0],
-                "total": values[1],
-            }
-            for key, values in sorted(by_operator_count.items())
-        },
-        "by_topic": {
-            key: {
-                "accuracy": values[0] / max(values[1], 1),
-                "correct": values[0],
-                "total": values[1],
-            }
-            for key, values in sorted(by_topic.items())
-        },
-    }
-    if dump_failures:
-        results["failure_summary"] = {
-            "|".join(key): value
-            for key, value in failure_summary.most_common()
-        }
-        results["failures"] = failures
-    return results
-
-
-def parse_args() -> argparse.Namespace:
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("--checkpoint", type=Path, default=Path("outputs/fusion_run/final_model"))
-    parser.add_argument("--data-path", type=Path, default=Path("arithmark_2.0.jsonl"))
-    parser.add_argument("--batch-size", type=int, default=64)
-    parser.add_argument("--device", default="auto")
-    parser.add_argument("--dtype", default="auto", choices=("auto", "float32", "bfloat16", "float16"))
-    parser.add_argument("--output", type=Path)
-    parser.add_argument(
-        "--max-examples",
-        type=int,
-        default=0,
-        help="Evaluate only the first N examples. Default evaluates all examples.",
-    )
-    parser.add_argument(
-        "--dump-failures",
-        action="store_true",
-        help="Include incorrectly scored examples and grouped failure summary.",
-    )
-    parser.add_argument(
-        "--failure-operator-count",
-        type=int,
-        default=None,
-        help="Only dump failures with this operator count, e.g. 1 for easy examples.",
-    )
-    parser.add_argument("--max-failures", type=int, default=100)
-    return parser.parse_args()
-
-
-def main() -> None:
-    args = parse_args()
-    if args.device == "auto":
-        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-    else:
-        device = torch.device(args.device)
-
-    data_path = ensure_data(args.data_path)
-    examples = load_examples(data_path, max_examples=args.max_examples)
-    dtype = None
-    if args.dtype == "float32":
-        dtype = torch.float32
-    elif args.dtype == "bfloat16":
-        dtype = torch.bfloat16
-    elif args.dtype == "float16":
-        dtype = torch.float16
-    model = AutoModelForCausalLM.from_pretrained(
-        args.checkpoint,
-        dtype=dtype,
-        trust_remote_code=True,
-    ).to(device)
-    tokenizer = AutoTokenizer.from_pretrained(args.checkpoint, trust_remote_code=True)
-    if tokenizer.pad_token_id is None:
-        tokenizer.pad_token = tokenizer.eos_token
-    results = evaluate(
-        model,
-        tokenizer,
-        examples,
-        device=device,
-        batch_size=args.batch_size,
-        dump_failures=args.dump_failures,
-        failure_operator_count=args.failure_operator_count,
-        max_failures=args.max_failures,
-    )
-    print(json.dumps(results, indent=2, sort_keys=True))
-    if args.output:
-        args.output.parent.mkdir(parents=True, exist_ok=True)
-        args.output.write_text(
-            json.dumps(results, indent=2, sort_keys=True) + "\n",
-            encoding="utf-8",
-        )
-
-
-if __name__ == "__main__":
-    main()

benchmark_results/lm_eval/results_2026-07-04T17-16-59.922847.json

@@ -1,355 +0,0 @@
-{
-  "results": {
-    "hellaswag": {
-      "name": "hellaswag",
-      "alias": "hellaswag",
-      "sample_len": 10042,
-      "acc,none": 0.2669786895040829,
-      "acc_stderr,none": 0.004414770331224705,
-      "acc_norm,none": 0.2700657239593706,
-      "acc_norm_stderr,none": 0.0044308610336166706
-    },
-    "arc_easy": {
-      "name": "arc_easy",
-      "alias": "arc_easy",
-      "sample_len": 2376,
-      "acc,none": 0.32154882154882153,
-      "acc_stderr,none": 0.009584091575640627,
-      "acc_norm,none": 0.31607744107744107,
-      "acc_norm_stderr,none": 0.009540440071928294
-    },
-    "arc_challenge": {
-      "name": "arc_challenge",
-      "alias": "arc_challenge",
-      "sample_len": 1172,
-      "acc,none": 0.17235494880546076,
-      "acc_stderr,none": 0.011037113093461295,
-      "acc_norm,none": 0.2098976109215017,
-      "acc_norm_stderr,none": 0.011900548748047421
-    },
-    "piqa": {
-      "name": "piqa",
-      "alias": "piqa",
-      "sample_len": 1838,
-      "acc,none": 0.5565832426550599,
-      "acc_stderr,none": 0.011590883373666865,
-      "acc_norm,none": 0.529923830250272,
-      "acc_norm_stderr,none": 0.011644913435420158
-    }
-  },
-  "group_subtasks": {},
-  "configs": {
-    "arc_challenge": {
-      "task": "arc_challenge",
-      "dataset_path": "allenai/ai2_arc",
-      "dataset_name": "ARC-Challenge",
-      "training_split": "train",
-      "validation_split": "validation",
-      "test_split": "test",
-      "doc_to_text": "Question: {{question}}\nAnswer:",
-      "doc_to_target": "{{choices.label.index(answerKey)}}",
-      "unsafe_code": false,
-      "doc_to_choice": "{{choices.text}}",
-      "description": "",
-      "target_delimiter": " ",
-      "fewshot_delimiter": "\n\n",
-      "fewshot_config": {
-        "sampler": "default",
-        "split": null,
-        "process_docs": null,
-        "fewshot_indices": null,
-        "samples": null,
-        "doc_to_text": "Question: {{question}}\nAnswer:",
-        "doc_to_choice": "{{choices.text}}",
-        "doc_to_target": "{{choices.label.index(answerKey)}}",
-        "gen_prefix": null,
-        "fewshot_delimiter": "\n\n",
-        "target_delimiter": " "
-      },
-      "num_fewshot": 0,
-      "metric_list": [
-        {
-          "metric": "acc",
-          "aggregation": "mean",
-          "higher_is_better": true
-        },
-        {
-          "metric": "acc_norm",
-          "aggregation": "mean",
-          "higher_is_better": true
-        }
-      ],
-      "output_type": "multiple_choice",
-      "repeats": 1,
-      "should_decontaminate": true,
-      "doc_to_decontamination_query": "Question: {{question}}\nAnswer:",
-      "metadata": {
-        "version": 1.0,
-        "pretrained": ".",
-        "trust_remote_code": true,
-        "dtype": "bfloat16",
-        "max_length": 548,
-        "config_source": "/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml"
-      }
-    },
-    "arc_easy": {
-      "task": "arc_easy",
-      "dataset_path": "allenai/ai2_arc",
-      "dataset_name": "ARC-Easy",
-      "training_split": "train",
-      "validation_split": "validation",
-      "test_split": "test",
-      "doc_to_text": "Question: {{question}}\nAnswer:",
-      "doc_to_target": "{{choices.label.index(answerKey)}}",
-      "unsafe_code": false,
-      "doc_to_choice": "{{choices.text}}",
-      "description": "",
-      "target_delimiter": " ",
-      "fewshot_delimiter": "\n\n",
-      "fewshot_config": {
-        "sampler": "default",
-        "split": null,
-        "process_docs": null,
-        "fewshot_indices": null,
-        "samples": null,
-        "doc_to_text": "Question: {{question}}\nAnswer:",
-        "doc_to_choice": "{{choices.text}}",
-        "doc_to_target": "{{choices.label.index(answerKey)}}",
-        "gen_prefix": null,
-        "fewshot_delimiter": "\n\n",
-        "target_delimiter": " "
-      },
-      "num_fewshot": 0,
-      "metric_list": [
-        {
-          "metric": "acc",
-          "aggregation": "mean",
-          "higher_is_better": true
-        },
-        {
-          "metric": "acc_norm",
-          "aggregation": "mean",
-          "higher_is_better": true
-        }
-      ],
-      "output_type": "multiple_choice",
-      "repeats": 1,
-      "should_decontaminate": true,
-      "doc_to_decontamination_query": "Question: {{question}}\nAnswer:",
-      "metadata": {
-        "version": 1.0,
-        "pretrained": ".",
-        "trust_remote_code": true,
-        "dtype": "bfloat16",
-        "max_length": 548,
-        "config_source": "/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml"
-      }
-    },
-    "hellaswag": {
-      "task": "hellaswag",
-      "dataset_path": "Rowan/hellaswag",
-      "training_split": "train",
-      "validation_split": "validation",
-      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
-      "doc_to_text": "{{query}}",
-      "doc_to_target": "{{label}}",
-      "unsafe_code": false,
-      "doc_to_choice": "choices",
-      "description": "",
-      "target_delimiter": " ",
-      "fewshot_delimiter": "\n\n",
-      "fewshot_config": {
-        "sampler": "default",
-        "split": null,
-        "process_docs": "<function process_docs at 0x77bb53ff80e0>",
-        "fewshot_indices": null,
-        "samples": null,
-        "doc_to_text": "{{query}}",
-        "doc_to_choice": "choices",
-        "doc_to_target": "{{label}}",
-        "gen_prefix": null,
-        "fewshot_delimiter": "\n\n",
-        "target_delimiter": " "
-      },
-      "num_fewshot": 0,
-      "metric_list": [
-        {
-          "metric": "acc",
-          "aggregation": "mean",
-          "higher_is_better": true
-        },
-        {
-          "metric": "acc_norm",
-          "aggregation": "mean",
-          "higher_is_better": true
-        }
-      ],
-      "output_type": "multiple_choice",
-      "repeats": 1,
-      "should_decontaminate": false,
-      "metadata": {
-        "version": 1.0,
-        "pretrained": ".",
-        "trust_remote_code": true,
-        "dtype": "bfloat16",
-        "max_length": 548,
-        "config_source": "/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml"
-      }
-    },
-    "piqa": {
-      "task": "piqa",
-      "dataset_path": "baber/piqa",
-      "training_split": "train",
-      "validation_split": "validation",
-      "doc_to_text": "Question: {{goal}}\nAnswer:",
-      "doc_to_target": "label",
-      "unsafe_code": false,
-      "doc_to_choice": "{{[sol1, sol2]}}",
-      "description": "",
-      "target_delimiter": " ",
-      "fewshot_delimiter": "\n\n",
-      "fewshot_config": {
-        "sampler": "default",
-        "split": null,
-        "process_docs": null,
-        "fewshot_indices": null,
-        "samples": null,
-        "doc_to_text": "Question: {{goal}}\nAnswer:",
-        "doc_to_choice": "{{[sol1, sol2]}}",
-        "doc_to_target": "label",
-        "gen_prefix": null,
-        "fewshot_delimiter": "\n\n",
-        "target_delimiter": " "
-      },
-      "num_fewshot": 0,
-      "metric_list": [
-        {
-          "metric": "acc",
-          "aggregation": "mean",
-          "higher_is_better": true
-        },
-        {
-          "metric": "acc_norm",
-          "aggregation": "mean",
-          "higher_is_better": true
-        }
-      ],
-      "output_type": "multiple_choice",
-      "repeats": 1,
-      "should_decontaminate": true,
-      "doc_to_decontamination_query": "goal",
-      "metadata": {
-        "version": 1.0,
-        "pretrained": ".",
-        "trust_remote_code": true,
-        "dtype": "bfloat16",
-        "max_length": 548,
-        "config_source": "/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml"
-      }
-    }
-  },
-  "versions": {
-    "arc_challenge": 1.0,
-    "arc_easy": 1.0,
-    "hellaswag": 1.0,
-    "piqa": 1.0
-  },
-  "n-shot": {
-    "arc_challenge": 0,
-    "arc_easy": 0,
-    "hellaswag": 0,
-    "piqa": 0
-  },
-  "higher_is_better": {
-    "arc_challenge": {
-      "acc": true,
-      "acc_norm": true
-    },
-    "arc_easy": {
-      "acc": true,
-      "acc_norm": true
-    },
-    "hellaswag": {
-      "acc": true,
-      "acc_norm": true
-    },
-    "piqa": {
-      "acc": true,
-      "acc_norm": true
-    }
-  },
-  "n-samples": {
-    "hellaswag": {
-      "original": 10042,
-      "effective": 10042
-    },
-    "arc_easy": {
-      "original": 2376,
-      "effective": 2376
-    },
-    "arc_challenge": {
-      "original": 1172,
-      "effective": 1172
-    },
-    "piqa": {
-      "original": 1838,
-      "effective": 1838
-    }
-  },
-  "config": {
-    "model": "hf",
-    "model_args": {
-      "pretrained": ".",
-      "trust_remote_code": true,
-      "dtype": "bfloat16",
-      "max_length": 548
-    },
-    "model_num_parameters": 2738880,
-    "model_dtype": "torch.float32",
-    "model_revision": "main",
-    "model_sha": "",
-    "batch_size": "auto:1",
-    "batch_sizes": [
-      64
-    ],
-    "device": "cuda:0",
-    "use_cache": null,
-    "limit": null,
-    "bootstrap_iters": 100000,
-    "gen_kwargs": {},
-    "random_seed": 0,
-    "numpy_seed": 1234,
-    "torch_seed": 1234,
-    "fewshot_seed": 1234
-  },
-  "git_hash": "f5846e8",
-  "date": 1783178146.3657796,
-  "pretty_env_info": "PyTorch version: 2.10.0+cu130\nIs debug build: False\nCUDA used to build PyTorch: 13.0\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14+deb12u1) 12.2.0\nClang version: Could not collect\nCMake version: version 3.25.1\nLibc version: glibc-2.36\n\nPython version: 3.11.2 (main, Apr  8 2026, 01:58:00) [GCC 12.2.0] (64-bit runtime)\nPython platform: Linux-6.18.33.2-microsoft-standard-WSL2-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: 13.2.51\nCUDA_MODULE_LOADING set to: \nGPU models and configuration: GPU 0: NVIDIA GeForce RTX 3080\nNvidia driver version: 596.36\ncuDNN version: Could not collect\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\nCaching allocator config: N/A\n\nCPU:\nArchitecture:                            x86_64\nCPU op-mode(s):                          32-bit, 64-bit\nAddress sizes:                           48 bits physical, 48 bits virtual\nByte Order:                              Little Endian\nCPU(s):                                  12\nOn-line CPU(s) list:                     0-11\nVendor ID:                               AuthenticAMD\nModel name:                              AMD Ryzen 5 7500F 6-Core Processor\nCPU family:                              25\nModel:                                   97\nThread(s) per core:                      2\nCore(s) per socket:                      6\nSocket(s):                               1\nStepping:                                2\nBogoMIPS:                                7399.82\nFlags:                                   fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology tsc_reliable nonstop_tsc cpuid extd_apicid tsc_known_freq pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy svm cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves avx512_bf16 clzero xsaveerptr arat npt nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avx512vbmi umip avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid fsrm\nVirtualization:                          AMD-V\nHypervisor vendor:                       Microsoft\nVirtualization type:                     full\nL1d cache:                               192 KiB (6 instances)\nL1i cache:                               192 KiB (6 instances)\nL2 cache:                                6 MiB (6 instances)\nL3 cache:                                32 MiB (1 instance)\nNUMA node(s):                            1\nNUMA node0 CPU(s):                       0-11\nVulnerability Gather data sampling:      Not affected\nVulnerability Ghostwrite:                Not affected\nVulnerability Indirect target selection: Not affected\nVulnerability Itlb multihit:             Not affected\nVulnerability L1tf:                      Not affected\nVulnerability Mds:                       Not affected\nVulnerability Meltdown:                  Not affected\nVulnerability Mmio stale data:           Not affected\nVulnerability Old microcode:             Not affected\nVulnerability Reg file data sampling:    Not affected\nVulnerability Retbleed:                  Not affected\nVulnerability Spec rstack overflow:      Vulnerable: Safe RET, no microcode\nVulnerability Spec store bypass:         Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:                Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:                Mitigation; Retpolines; IBPB conditional; IBRS_FW; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                     Not affected\nVulnerability Tsa:                       Vulnerable: No microcode\nVulnerability Tsx async abort:           Not affected\nVulnerability Vmscape:                   Not affected\n\nVersions of relevant libraries:\n[pip3] flash_attn==2.8.3+cu130torch2.10\n[pip3] mypy_extensions==1.1.0\n[pip3] numpy==2.2.6\n[pip3] nvidia-cublas==13.1.0.3\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti==13.0.85\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc==13.0.88\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime==13.0.96\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cudnn-cu13==9.15.1.9\n[pip3] nvidia-cudnn-frontend==1.18.0\n[pip3] nvidia-cufft==12.0.0.61\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand==10.4.0.35\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver==12.0.4.66\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse==12.6.3.3\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-cusparselt-cu13==0.8.0\n[pip3] nvidia-nccl-cu12==2.27.5\n[pip3] nvidia-nccl-cu13==2.28.9\n[pip3] nvidia-nvjitlink==13.0.88\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx==13.0.85\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] onnxruntime==1.20.1\n[pip3] rapidocr-onnxruntime==1.4.4\n[pip3] rotary-embedding-torch==0.8.8\n[pip3] torch==2.10.0+cu130\n[pip3] torch_c_dlpack_ext==0.1.5\n[pip3] torch-lr-finder==0.2.2\n[pip3] torch-tb-profiler==0.4.3\n[pip3] torchaudio==2.10.0+cu130\n[pip3] torchmetrics==1.9.0\n[pip3] torchvision==0.25.0+cu130\n[pip3] triton==3.6.0\n[conda] Could not collect",
-  "transformers_version": "4.57.6",
-  "lm_eval_version": "0.4.12",
-  "upper_git_hash": "f5846e8ab83c0da3653a1f7a04d470911c8f5065",
-  "tokenizer_pad_token": [
-    "<|pad|>",
-    "0"
-  ],
-  "tokenizer_eos_token": [
-    "<|eos|>",
-    "2"
-  ],
-  "tokenizer_bos_token": [
-    "<|bos|>",
-    "1"
-  ],
-  "eot_token_id": 2,
-  "max_length": 548,
-  "task_hashes": {},
-  "model_source": "hf",
-  "model_name": ".",
-  "model_name_sanitized": ".",
-  "system_instruction": null,
-  "system_instruction_sha": null,
-  "fewshot_as_multiturn": null,
-  "chat_template": null,
-  "chat_template_sha": null,
-  "total_evaluation_time_seconds": "80.63422504799973"
-}

lm_eval_fusion

@@ -1,9 +0,0 @@
-#!/usr/bin/env python
-"""Run lm-eval with the local Atom2.7m model registered."""
-
-import lm_eval_fusion  # noqa: F401
-from lm_eval.__main__ import cli_evaluate
-
-
-if __name__ == "__main__":
-    cli_evaluate()

lm_eval_fusion.py

@@ -1,299 +0,0 @@
-"""lm-eval wrapper for Atom2.7m checkpoints.
-
-The standard ``hf`` lm-eval model does not use the fusion tokenizer wrapper and
-does not pass arithmetic feature streams. This model keeps lm-eval's
-log-likelihood interface while encoding with ``tokenizer_utils.load_tokenizer``
-and forwarding ``place_ids`` and ``role_ids``.
-"""
-
-from __future__ import annotations
-
-from contextlib import nullcontext
-from pathlib import Path
-from typing import Any
-
-import torch
-import torch.nn.functional as F
-from lm_eval.api.model import LM
-from lm_eval.api.registry import register_model
-from tqdm import tqdm
-from transformers import AutoModelForCausalLM
-
-from tokenizer_utils import EOT_ID, FusionTokenizer, load_tokenizer
-
-
-def _parse_bool(value: Any, default: bool = False) -> bool:
-    if value is None:
-        return default
-    if isinstance(value, bool):
-        return value
-    return str(value).strip().lower() in {"1", "true", "yes", "on"}
-
-
-def _parse_batch_size(value: int | str | None, max_batch_size: int | None) -> int:
-    if value is None:
-        return 1
-    if isinstance(value, int):
-        return value
-    text = str(value).strip().lower()
-    if text == "auto" or text.startswith("auto:"):
-        return int(max_batch_size or 64)
-    return int(text)
-
-
-def _dtype_from_name(value: str | torch.dtype | None) -> torch.dtype | None:
-    if value is None or value == "auto":
-        return None
-    if isinstance(value, torch.dtype):
-        return value
-    normalized = str(value).replace("torch.", "").lower()
-    if normalized in {"bf16", "bfloat16"}:
-        return torch.bfloat16
-    if normalized in {"fp16", "float16", "half"}:
-        return torch.float16
-    if normalized in {"fp32", "float32", "float"}:
-        return torch.float32
-    raise ValueError(f"Unsupported dtype: {value!r}")
-
-
-@register_model("atom2.7m")
-class FusionGPTLM(LM):
-    """Fusion-tokenizer GPT adapter for lm-eval log-likelihood tasks."""
-
-    def __init__(
-        self,
-        pretrained: str = "outputs/fusion_run/final_model",
-        tokenizer_dir: str = "tokenizer_4k",
-        batch_size: int | str | None = 1,
-        max_batch_size: int | None = 64,
-        max_length: int | None = None,
-        device: str | None = "cuda",
-        dtype: str | torch.dtype | None = "auto",
-        mixed_precision_dtype: str | torch.dtype | None = "auto",
-        trust_remote_code: bool | str | None = None,
-        **_: Any,
-    ) -> None:
-        super().__init__()
-        del trust_remote_code
-        if device is None or device == "auto":
-            device = "cuda" if torch.cuda.is_available() else "cpu"
-        self._device = torch.device(device)
-        self.batch_size = _parse_batch_size(batch_size, max_batch_size)
-        self.tokenizer: FusionTokenizer = load_tokenizer(Path(tokenizer_dir))
-        self.model = AutoModelForCausalLM.from_pretrained(
-            Path(pretrained),
-            trust_remote_code=True,
-        ).to(self.device)
-        model_dtype = _dtype_from_name(dtype)
-        if model_dtype is not None:
-            self.model = self.model.to(dtype=model_dtype)
-        if mixed_precision_dtype == "auto":
-            self.mixed_precision_dtype = (
-                torch.bfloat16 if self.device.type == "cuda" else None
-            )
-        else:
-            self.mixed_precision_dtype = _dtype_from_name(mixed_precision_dtype)
-        self.model.eval()
-        self.max_length = int(
-            max_length
-            or getattr(self.model.config, "block_size", None)
-            or getattr(self.model.config, "max_position_embeddings", 512)
-        )
-
-    @property
-    def eot_token_id(self) -> int:
-        return EOT_ID
-
-    def tok_encode(
-        self,
-        string: str,
-        add_special_tokens: bool | None = None,
-        left_truncate_len: int | None = None,
-        **_: Any,
-    ) -> list[int]:
-        del add_special_tokens
-        ids = self.tokenizer.encode(string).input_ids
-        if left_truncate_len is not None:
-            ids = ids[-left_truncate_len:]
-        return ids
-
-    def tok_decode(self, tokens, skip_special_tokens: bool = True) -> str:
-        if isinstance(tokens, int):
-            tokens = [tokens]
-        return self.tokenizer.decode(tokens, skip_special_tokens=skip_special_tokens)
-
-    def _encode_request(
-        self,
-        context: str,
-        continuation: str,
-    ) -> tuple[list[int], list[int], list[int], list[int], int]:
-        if context == "":
-            continuation_encoding = self.tokenizer.encode(continuation)
-            ids = [self.eot_token_id] + continuation_encoding.input_ids
-            place_ids = [0] + continuation_encoding.place_ids
-            role_ids = [0] + continuation_encoding.role_ids
-            context_len = 1
-            continuation_ids = continuation_encoding.input_ids
-        else:
-            n_spaces = len(context) - len(context.rstrip())
-            if n_spaces > 0:
-                continuation = context[-n_spaces:] + continuation
-                context = context[:-n_spaces]
-            full_encoding = self.tokenizer.encode(context + continuation)
-            context_encoding = self.tokenizer.encode(context)
-            ids = full_encoding.input_ids
-            place_ids = full_encoding.place_ids
-            role_ids = full_encoding.role_ids
-            context_len = len(context_encoding.input_ids)
-            continuation_ids = ids[context_len:]
-
-        if not continuation_ids:
-            raise ValueError("Continuation encoded to zero tokens")
-        return ids, place_ids, role_ids, continuation_ids, context_len
-
-    def loglikelihood(
-        self,
-        requests: list["Instance"],
-        disable_tqdm: bool = False,
-    ) -> list[tuple[float, bool]]:
-        encoded = [
-            self._encode_request(context, continuation)
-            for context, continuation in tqdm(
-                [req.args for req in requests],
-                desc="Fusion tokenizing inputs",
-                disable=disable_tqdm,
-            )
-        ]
-        results: list[tuple[float, bool]] = []
-        for start in tqdm(
-            range(0, len(encoded), self.batch_size),
-            desc="Running fusion loglikelihood requests",
-            disable=disable_tqdm or self.rank != 0,
-        ):
-            batch = encoded[start : start + self.batch_size]
-            rows = []
-            row_places = []
-            row_roles = []
-            row_targets = []
-            row_score_slices = []
-            for ids, place_ids, role_ids, continuation_ids, context_len in batch:
-                window_start = max(0, len(ids) - (self.max_length + 1))
-                window_ids = ids[window_start:]
-                window_places = place_ids[window_start:]
-                window_roles = role_ids[window_start:]
-                input_ids = window_ids[:-1]
-                targets = window_ids[1:]
-                full_score_start = context_len - 1
-                full_score_end = len(ids) - 1
-                score_start = max(full_score_start, window_start) - window_start
-                score_end = full_score_end - window_start
-                if score_end <= score_start:
-                    raise ValueError("No continuation tokens remain after truncation")
-                scored_continuation_ids = continuation_ids[-(score_end - score_start) :]
-                rows.append(input_ids)
-                row_places.append(window_places[:-1])
-                row_roles.append(window_roles[:-1])
-                row_targets.append(targets)
-                row_score_slices.append((score_start, score_end, scored_continuation_ids))
-
-            max_len = max(len(row) for row in rows)
-            input_tensor = torch.full(
-                (len(rows), max_len),
-                self.eot_token_id,
-                dtype=torch.long,
-                device=self.device,
-            )
-            place_tensor = torch.zeros_like(input_tensor)
-            role_tensor = torch.zeros_like(input_tensor)
-            attention_mask = torch.zeros_like(input_tensor, dtype=torch.bool)
-            target_tensor = torch.full_like(input_tensor, self.eot_token_id)
-            for row, (ids, places, roles, targets) in enumerate(
-                zip(rows, row_places, row_roles, row_targets, strict=True)
-            ):
-                length = len(ids)
-                input_tensor[row, :length] = torch.tensor(ids, device=self.device)
-                place_tensor[row, :length] = torch.tensor(places, device=self.device)
-                role_tensor[row, :length] = torch.tensor(roles, device=self.device)
-                target_tensor[row, :length] = torch.tensor(targets, device=self.device)
-                attention_mask[row, :length] = True
-
-            autocast = (
-                torch.autocast(
-                    device_type=self.device.type,
-                    dtype=self.mixed_precision_dtype,
-                    enabled=self.mixed_precision_dtype is not None,
-                )
-                if self.device.type == "cuda"
-                else nullcontext()
-            )
-            with torch.inference_mode(), autocast:
-                logits = self.model(
-                    input_ids=input_tensor,
-                    place_ids=place_tensor,
-                    role_ids=role_tensor,
-                    attention_mask=attention_mask,
-                ).logits
-                log_probs = F.log_softmax(logits.float(), dim=-1)
-
-            for row, (score_start, score_end, continuation_ids) in enumerate(row_score_slices):
-                row_log_probs = log_probs[row, score_start:score_end]
-                row_targets_for_score = target_tensor[row, score_start:score_end]
-                token_log_probs = torch.gather(
-                    row_log_probs,
-                    1,
-                    row_targets_for_score.unsqueeze(-1),
-                ).squeeze(-1)
-                greedy = torch.equal(
-                    row_log_probs.argmax(dim=-1),
-                    torch.tensor(continuation_ids, dtype=torch.long, device=self.device),
-                )
-                results.append((float(token_log_probs.sum().item()), bool(greedy)))
-
-        return results
-
-    def loglikelihood_rolling(
-        self,
-        requests: list["Instance"],
-        disable_tqdm: bool = False,
-    ) -> list[float]:
-        results = []
-        for (text,) in tqdm(
-            [req.args for req in requests],
-            desc="Running fusion rolling loglikelihood",
-            disable=disable_tqdm or self.rank != 0,
-        ):
-            encoding = self.tokenizer.encode(text)
-            ids = encoding.input_ids
-            places = encoding.place_ids
-            roles = encoding.role_ids
-            total = 0.0
-            start = 0
-            while start < len(ids):
-                end = min(len(ids), start + self.max_length)
-                prefix = [self.eot_token_id] if start == 0 else ids[start - 1 : start]
-                chunk_ids = prefix + ids[start:end]
-                chunk_places = [0] + places[start:end] if start == 0 else places[start - 1 : end]
-                chunk_roles = [0] + roles[start:end] if start == 0 else roles[start - 1 : end]
-                input_ids = torch.tensor([chunk_ids[:-1]], dtype=torch.long, device=self.device)
-                place_ids = torch.tensor([chunk_places[:-1]], dtype=torch.long, device=self.device)
-                role_ids = torch.tensor([chunk_roles[:-1]], dtype=torch.long, device=self.device)
-                targets = torch.tensor(chunk_ids[1:], dtype=torch.long, device=self.device)
-                with torch.inference_mode():
-                    logits = self.model(
-                        input_ids=input_ids,
-                        place_ids=place_ids,
-                        role_ids=role_ids,
-                    ).logits[0]
-                    log_probs = F.log_softmax(logits.float(), dim=-1)
-                total += float(
-                    torch.gather(log_probs, 1, targets.unsqueeze(-1)).sum().item()
-                )
-                start = end
-            results.append(total)
-        return results
-
-    def generate_until(self, requests, disable_tqdm: bool = False) -> list[str]:
-        raise NotImplementedError(
-            "FusionGPTLM currently supports loglikelihood tasks. "
-            "Use tasks with multiple-choice/loglikelihood output."
-        )