Fix: RoPE inv buffer could fail to materialize under some transformers meta-device loading paths, corrupting all attention output (see repo history for full diagnosis)

modeling_char_bert.py

@@ -34,11 +34,18 @@ class RMSNorm(nn.Module):
 class RoPE(nn.Module):
     def __init__(self, dim, base=10000.0):
         super().__init__()
-        inv = 1.0 / (base ** (torch.arange(0, dim, 2).float() / dim))
-        self.register_buffer("inv", inv, persistent=False)
+        self.dim = dim
+        self.base = base
 
     def cos_sin(self, pos):
-        f = torch.outer(pos.float(), self.inv)
+        # Recomputed on every call rather than cached in a registered buffer: a
+        # persistent=False buffer is never covered by the checkpoint's state dict,
+        # so it depends entirely on __init__-time materialization -- which some
+        # transformers versions' meta-device/low_cpu_mem_usage loading path can
+        # skip, silently leaving this tensor uninitialized. Recomputing here is
+        # immune to that regardless of how the model was constructed/loaded.
+        inv = 1.0 / (self.base ** (torch.arange(0, self.dim, 2, device=pos.device).float() / self.dim))
+        f = torch.outer(pos.float(), inv)
         emb = torch.cat([f, f], -1)
         return emb.cos(), emb.sin()