Upload folder using huggingface_hub

code/inference.py

@@ -148,11 +148,17 @@ class SpeechLLMForInference:
         # SpeechLLM.encode_audio casts features to the encoder's own dtype
         batch["input_features"] = batch["input_features"].to(self.device)
 
+        # The trainable tensors (connector, LoRA) stay in float32 while the frozen encoder and
+        # LLM run in `dtype`, exactly as in training -- so the forward pass has to happen under
+        # autocast, or layer_norm sees a float32 weight and a float16 activation and raises
+        # "expected scalar type Half but found Float".
         # passing inputs_embeds means `generate` returns only the new tokens --
         # there is no prompt prefix to slice off
-        generated = self.model.generate(
-            batch, self.tokenizer, max_new_tokens=max_new_tokens,
-            do_sample=do_sample, **generation_kwargs)
+        with torch.autocast(self.device.split(":")[0], dtype=self.dtype,
+                            enabled=not self.device.startswith("cpu")):
+            generated = self.model.generate(
+                batch, self.tokenizer, max_new_tokens=max_new_tokens,
+                do_sample=do_sample, **generation_kwargs)
         return [t.strip() for t in
                 self.tokenizer.batch_decode(generated, skip_special_tokens=True)]