modeling_custom_sequence_classifier.py

import torch
import torch.nn as nn

from transformers import AutoModel, PreTrainedModel
from transformers.modeling_outputs import SequenceClassifierOutput


class CustomSequenceClassificationModel(PreTrainedModel):
    """Sequence classification model with a configurable GELU + dropout dense head."""

    def __init__(self, config):
        super().__init__(config)
        self.num_labels = config.num_labels
        self.config = config
        self.backbone = AutoModel.from_config(config)

        hidden_size = getattr(config, "hidden_size", None)
        if hidden_size is None:
            raise ValueError(
                "Backbone config is missing 'hidden_size'; cannot build custom head."
            )

        # Primary config keys for Hub reloads.
        # Fallback keeps compatibility with previously saved configs.
        head_layers = getattr(
            config, "head_layers", getattr(config, "classifier_head_layers", 1)
        )
        head_dropout = getattr(
            config, "head_dropout", getattr(config, "classifier_head_dropout", 0.1)
        )
        self.head_layers = int(head_layers)
        self.head_dropout = float(head_dropout)

        self.classifier = self._build_head(
            input_dim=hidden_size,
            output_dim=self.num_labels,
            head_layers=self.head_layers,
            head_dropout=self.head_dropout,
        )
        self.post_init()

    @staticmethod
    def _build_head(
        input_dim: int, output_dim: int, head_layers: int, head_dropout: float
    ) -> nn.Module:
        if head_layers < 1:
            raise ValueError("head_layers must be >= 1")

        # head_layers includes the final projection layer.
        # For head_layers=1 this is equivalent to a single linear classifier.
        if head_layers == 1:
            return nn.Linear(input_dim, output_dim)

        layers = []
        for _ in range(head_layers - 1):
            layers.extend(
                [
                    nn.Linear(input_dim, input_dim),
                    nn.GELU(),
                    nn.Dropout(head_dropout),
                ]
            )
        layers.append(nn.Linear(input_dim, output_dim))
        return nn.Sequential(*layers)

    def _pooled_representation(self, outputs) -> torch.Tensor:
        if hasattr(outputs, "pooler_output") and outputs.pooler_output is not None:
            return outputs.pooler_output
        return outputs.last_hidden_state[:, 0]

    def forward(
        self,
        input_ids=None,
        attention_mask=None,
        token_type_ids=None,
        labels=None,
        **kwargs,
    ):
        backbone_inputs = {
            "input_ids": input_ids,
            "attention_mask": attention_mask,
            **kwargs,
        }
        if token_type_ids is not None:
            backbone_inputs["token_type_ids"] = token_type_ids

        backbone_outputs = self.backbone(**backbone_inputs)

        pooled = self._pooled_representation(backbone_outputs)
        logits = self.classifier(pooled)

        loss = None
        if labels is not None:
            if self.config.problem_type is None:
                if self.num_labels == 1:
                    self.config.problem_type = "regression"
                elif torch.is_floating_point(labels):
                    self.config.problem_type = "multi_label_classification"
                else:
                    self.config.problem_type = "single_label_classification"

            if self.config.problem_type == "regression":
                loss_fct = nn.MSELoss()
                loss = loss_fct(logits.squeeze(), labels.squeeze())
            elif self.config.problem_type == "single_label_classification":
                loss_fct = nn.CrossEntropyLoss()
                loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
            else:
                loss_fct = nn.BCEWithLogitsLoss()
                loss = loss_fct(logits, labels)

        return SequenceClassifierOutput(
            loss=loss,
            logits=logits,
            hidden_states=backbone_outputs.hidden_states,
            attentions=backbone_outputs.attentions,
        )