Commit ·
581b2f2
0
Parent(s):
Duplicate from Edge0/Audio8-ASR-Infinite
Browse filesCo-authored-by: wanglamao <wanglamao@users.noreply.huggingface.co>
- .gitattributes +36 -0
- README.md +193 -0
- chat_template.jinja +54 -0
- config.json +169 -0
- configuration_audio8_asr_infinite.py +230 -0
- generation_config.json +12 -0
- model.safetensors +3 -0
- model.safetensors.index.json +945 -0
- modeling_audio8_asr_infinite.py +1338 -0
- preprocessor_config.json +12 -0
- semantic_vad_heads.safetensors +3 -0
- tokenizer.json +3 -0
- tokenizer_config.json +21 -0
.gitattributes
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
*.7z filter=lfs diff=lfs merge=lfs -text
|
| 2 |
+
*.arrow filter=lfs diff=lfs merge=lfs -text
|
| 3 |
+
*.bin filter=lfs diff=lfs merge=lfs -text
|
| 4 |
+
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
+
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 6 |
+
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
+
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
+
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
+
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
+
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
+
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
+
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
+
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
+
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
+
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
+
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
+
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
+
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
+
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
+
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
+
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
+
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
+
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
+
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 25 |
+
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
+
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 27 |
+
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
+
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
+
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
+
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
+
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
+
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
+
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
+
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
+
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
README.md
ADDED
|
@@ -0,0 +1,193 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: apache-2.0
|
| 3 |
+
language:
|
| 4 |
+
- zh
|
| 5 |
+
- en
|
| 6 |
+
library_name: transformers
|
| 7 |
+
pipeline_tag: automatic-speech-recognition
|
| 8 |
+
tags:
|
| 9 |
+
- streaming
|
| 10 |
+
- realtime
|
| 11 |
+
- speech-recognition
|
| 12 |
+
- audio
|
| 13 |
+
---
|
| 14 |
+
|
| 15 |
+
<div align="center">
|
| 16 |
+
|
| 17 |
+
# Audio8 ASR Infinite
|
| 18 |
+
|
| 19 |
+
[](https://huggingface.co/Edge0/Audio8-ASR-Infinite)
|
| 20 |
+
[](https://github.com/Edge0-AI/Audio8-ASR-Infinite)
|
| 21 |
+
[](https://github.com/Edge0-AI/Audio8-ASR-Infinite)
|
| 22 |
+
[](https://github.com/Edge0-AI/Audio8-ASR-Infinite/blob/main/LICENSE)
|
| 23 |
+
|
| 24 |
+
</div>
|
| 25 |
+
|
| 26 |
+
**Audio8 ASR Infinite** is a native streaming speech recognition model built to be
|
| 27 |
+
as responsive as possible. It offers a selectable audio clock (80/120/160 ms) and
|
| 28 |
+
a transcription delay (240–560 ms).
|
| 29 |
+
With our adapted vLLM build it transcribes unlimited-length audio **24/7** without drifting.
|
| 30 |
+
|
| 31 |
+
## Highlights
|
| 32 |
+
|
| 33 |
+
- **Super responsive** — the native streaming architecture decodes 12.5 times per second.
|
| 34 |
+
- **Unlimited-length transcription** — a rolling KV cache keeps memory and
|
| 35 |
+
latency bounded, even in **24/7 operation**.
|
| 36 |
+
- **Selectable streaming clock** — one text token per clock step
|
| 37 |
+
(12.5 / 8.3 / 6.25 decisions per second), balancing perception granularity and resource cost.
|
| 38 |
+
- **Configurable transcription delay** — set how much delay to trade for accuracy.
|
| 39 |
+
- **Semantic VAD** — distinguishes thinking pauses, stuttering and real end of turn, where traditional acoustic VAD fails.
|
| 40 |
+
- **Bilingual** — Chinese and English.
|
| 41 |
+
|
| 42 |
+
## Optimized operation points
|
| 43 |
+
|
| 44 |
+
The following combinations of frame length and delay are post-trained. Other combinations can be used but performance may not be optimum.
|
| 45 |
+
|
| 46 |
+
| audio clock | `frame_len` | `streaming_n_left_pad_tokens` | selectable `target_delay_ms` |
|
| 47 |
+
| --- | --- | --- | --- |
|
| 48 |
+
| 80 ms | 4 | 18 | 240 / 320 / 480 / 560 |
|
| 49 |
+
| 120 ms | 6 | 12 | 240 / 480 |
|
| 50 |
+
| 160 ms | 8 | 9 | 320 / 480 |
|
| 51 |
+
|
| 52 |
+
`target_delay_ms` must be an integer multiple of the selected clock, so longer
|
| 53 |
+
delays stay available at every clock even when they are not listed above.
|
| 54 |
+
|
| 55 |
+
## Architecture
|
| 56 |
+
|
| 57 |
+
Inherits the Voxtral realtime audio architecture and DSM-style streaming.
|
| 58 |
+
|
| 59 |
+
| Component | Initial weights | Trained |
|
| 60 |
+
| --- | --- | --- |
|
| 61 |
+
| Causal Audio Tower | Voxtral Realtime 4B | ✅ |
|
| 62 |
+
| Audio Projector | random initialisation | ✅ |
|
| 63 |
+
| Frame Length Embedding | random initialisation | ✅ |
|
| 64 |
+
| Decoder | Qwen2.5-3B-Instruct | ✅ |
|
| 65 |
+
| LM Head | Qwen2.5-3B-Instruct | ✅ |
|
| 66 |
+
|
| 67 |
+
Checkpoint specification:
|
| 68 |
+
|
| 69 |
+
| | |
|
| 70 |
+
| --- | --- |
|
| 71 |
+
| audio tower | 32 layers, hidden 1280, 128 mel bins, sliding window 750 |
|
| 72 |
+
| text decoder | 36 layers, hidden 2048, 16 query heads / 2 KV heads |
|
| 73 |
+
| projector | max frame len 8 → projection size 10240, gelu |
|
| 74 |
+
| frame-length conditioning | enabled (`use_frame_len_embedding: true`) |
|
| 75 |
+
| semantic VAD heads | `semantic_vad_heads.safetensors`, 8 classes, horizons 0.5 / 1.0 / 2.0 / 3.0 s |
|
| 76 |
+
| vocab size | 151936 |
|
| 77 |
+
| dtype | bfloat16 |
|
| 78 |
+
| weights | 8.17 GB `model.safetensors` (+ `semantic_vad_heads.safetensors`) |
|
| 79 |
+
|
| 80 |
+
## Roadmap
|
| 81 |
+
|
| 82 |
+
This is the **preview release**: it delivers the transcription base. Realtime
|
| 83 |
+
semantic perception is being built on the same frame grid and the same acoustic
|
| 84 |
+
forward pass.
|
| 85 |
+
|
| 86 |
+
| Stage | Status | Scope |
|
| 87 |
+
| --- | --- | --- |
|
| 88 |
+
| **Preview — ASR base** | ✅ done | Streaming Chinese/English transcription: selectable 80/120/160 ms clock, configurable `target_delay_ms`, unlimited-length rolling KV window |
|
| 89 |
+
| **Formal release** | 🏃in progress | Frame-level semantic perception on the same grid, beyond transcription |
|
| 90 |
+
|
| 91 |
+
## Evaluation
|
| 92 |
+
|
| 93 |
+
### 480 ms Delay, 80ms frame length
|
| 94 |
+
|
| 95 |
+
| test set | metric | Audio8 ASR Infinite | Voxtral-Mini-4B-Realtime-2602 | nemotron-3.5-asr-streaming-0.6b |
|
| 96 |
+
| --- | --- | --- | --- | --- |
|
| 97 |
+
| aishell1/test | CER | **1.750** | 16.795 | 12.927@560ms |
|
| 98 |
+
| aishell4/test | CER | **2.893** | 16.456 | 14.677@560ms |
|
| 99 |
+
| librispeech test.clean | WER | 3.042 | **2.210** | 3.353@560ms |
|
| 100 |
+
| librispeech test.other | WER | 6.808 | **5.552** | 7.140@560ms |
|
| 101 |
+
| **average** | | **3.623** | 10.253 (2 sets) | 9.524 |
|
| 102 |
+
|
| 103 |
+
Greedy decode with EOS suppressed, at the 80 ms audio clock with
|
| 104 |
+
`target_delay_ms = 480` (6 delay tokens). Error rates in percent. No repetition
|
| 105 |
+
loops and no dropped trailing words.
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
## Usage
|
| 109 |
+
|
| 110 |
+
Programmatic simulated-streaming decode with the embedded remote code:
|
| 111 |
+
|
| 112 |
+
```python
|
| 113 |
+
import numpy as np
|
| 114 |
+
import torch
|
| 115 |
+
from transformers import AutoFeatureExtractor, AutoTokenizer
|
| 116 |
+
|
| 117 |
+
from audio8_asr_infinite.modeling.modeling_audio8_asr_infinite import (
|
| 118 |
+
Audio8ASRInfiniteForConditionalGeneration,
|
| 119 |
+
resolve_qwen_language_token_id,
|
| 120 |
+
resolve_qwen_streaming_special_token_ids,
|
| 121 |
+
)
|
| 122 |
+
from audio8_asr_infinite.streaming_inference import simulated_streaming_greedy_decode_batch
|
| 123 |
+
|
| 124 |
+
checkpoint = "Edge0/Audio8-ASR-Infinite"
|
| 125 |
+
tokenizer = AutoTokenizer.from_pretrained(checkpoint, trust_remote_code=True)
|
| 126 |
+
feature_extractor = AutoFeatureExtractor.from_pretrained(checkpoint, trust_remote_code=True)
|
| 127 |
+
model = Audio8ASRInfiniteForConditionalGeneration.from_pretrained(
|
| 128 |
+
checkpoint, trust_remote_code=True, torch_dtype=torch.bfloat16
|
| 129 |
+
).eval().cuda()
|
| 130 |
+
|
| 131 |
+
class AudioConfig: # duck-typed: raw_audio_samples_per_token / streaming_n_left_pad_tokens / sampling_rate
|
| 132 |
+
raw_audio_samples_per_token = 1280 # 80 ms @ 16 kHz
|
| 133 |
+
streaming_n_left_pad_tokens = 18
|
| 134 |
+
sampling_rate = 16000
|
| 135 |
+
|
| 136 |
+
waveform = np.load("sample.npy", allow_pickle=False).astype(np.float32) # [-1, 1], 16 kHz mono
|
| 137 |
+
results = simulated_streaming_greedy_decode_batch(
|
| 138 |
+
model=model,
|
| 139 |
+
tokenizer=tokenizer,
|
| 140 |
+
feature_extractor=feature_extractor,
|
| 141 |
+
waveforms=[waveform],
|
| 142 |
+
language_token_ids=[resolve_qwen_language_token_id(tokenizer, "zh")],
|
| 143 |
+
special_ids=resolve_qwen_streaming_special_token_ids(tokenizer),
|
| 144 |
+
audio_config=AudioConfig(),
|
| 145 |
+
num_delay_tokens=[480 // 80],
|
| 146 |
+
right_pad_text_tokens=10,
|
| 147 |
+
dtype=torch.bfloat16,
|
| 148 |
+
device=next(model.parameters()).device,
|
| 149 |
+
max_new_tokens=512,
|
| 150 |
+
)
|
| 151 |
+
print(results[0]["final_text"])
|
| 152 |
+
```
|
| 153 |
+
|
| 154 |
+
Only a full merged weight directory is supported (this repository as-is);
|
| 155 |
+
adapter-style or partially converted weights are not.
|
| 156 |
+
|
| 157 |
+
## 24/7 inference with vLLM
|
| 158 |
+
|
| 159 |
+
Docker compose is the canonical deployment path; it also serves the web demo:
|
| 160 |
+
|
| 161 |
+
```bash
|
| 162 |
+
cd docker
|
| 163 |
+
AUDIO8_MODEL_DIR=/path/to/checkpoint docker compose up -d
|
| 164 |
+
```
|
| 165 |
+
|
| 166 |
+
Verify with the web client shipped in the same stack:
|
| 167 |
+
|
| 168 |
+
```
|
| 169 |
+
http://localhost:8080/ # plain HTTP
|
| 170 |
+
https://localhost:8443/ # TLS proxy; accept the self-signed certificate
|
| 171 |
+
```
|
| 172 |
+
|
| 173 |
+
The same socket can be driven from a terminal:
|
| 174 |
+
|
| 175 |
+
```bash
|
| 176 |
+
python -m audio8_asr_infinite.examples.vllm_realtime_client \
|
| 177 |
+
--ws-url ws://127.0.0.1:18191/v1/realtime \
|
| 178 |
+
--audio sample.wav --language zh --target-delay-ms 480 --pace
|
| 179 |
+
```
|
| 180 |
+
|
| 181 |
+
`18191` is the host port published by `docker/docker-compose.yml`; the service
|
| 182 |
+
itself listens on `18190` inside the compose network. The rolling KV window is
|
| 183 |
+
30 s with exact RoPE re-basing, which is what keeps memory and latency bounded
|
| 184 |
+
over 24/7 operation.
|
| 185 |
+
|
| 186 |
+
## Torch inference (simulated streaming decode)
|
| 187 |
+
|
| 188 |
+
```bash
|
| 189 |
+
python -m audio8_asr_infinite.examples.torch_streaming_decode \
|
| 190 |
+
--checkpoint /path/to/checkpoint \
|
| 191 |
+
--audio sample.wav --language zh --transcription-delay-ms 480
|
| 192 |
+
```
|
| 193 |
+
|
chat_template.jinja
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{%- if tools %}
|
| 2 |
+
{{- '<|im_start|>system\n' }}
|
| 3 |
+
{%- if messages[0]['role'] == 'system' %}
|
| 4 |
+
{{- messages[0]['content'] }}
|
| 5 |
+
{%- else %}
|
| 6 |
+
{{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
|
| 7 |
+
{%- endif %}
|
| 8 |
+
{{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
|
| 9 |
+
{%- for tool in tools %}
|
| 10 |
+
{{- "\n" }}
|
| 11 |
+
{{- tool | tojson }}
|
| 12 |
+
{%- endfor %}
|
| 13 |
+
{{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
|
| 14 |
+
{%- else %}
|
| 15 |
+
{%- if messages[0]['role'] == 'system' %}
|
| 16 |
+
{{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
|
| 17 |
+
{%- else %}
|
| 18 |
+
{{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
|
| 19 |
+
{%- endif %}
|
| 20 |
+
{%- endif %}
|
| 21 |
+
{%- for message in messages %}
|
| 22 |
+
{%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
|
| 23 |
+
{{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
|
| 24 |
+
{%- elif message.role == "assistant" %}
|
| 25 |
+
{{- '<|im_start|>' + message.role }}
|
| 26 |
+
{%- if message.content %}
|
| 27 |
+
{{- '\n' + message.content }}
|
| 28 |
+
{%- endif %}
|
| 29 |
+
{%- for tool_call in message.tool_calls %}
|
| 30 |
+
{%- if tool_call.function is defined %}
|
| 31 |
+
{%- set tool_call = tool_call.function %}
|
| 32 |
+
{%- endif %}
|
| 33 |
+
{{- '\n<tool_call>\n{"name": "' }}
|
| 34 |
+
{{- tool_call.name }}
|
| 35 |
+
{{- '", "arguments": ' }}
|
| 36 |
+
{{- tool_call.arguments | tojson }}
|
| 37 |
+
{{- '}\n</tool_call>' }}
|
| 38 |
+
{%- endfor %}
|
| 39 |
+
{{- '<|im_end|>\n' }}
|
| 40 |
+
{%- elif message.role == "tool" %}
|
| 41 |
+
{%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
|
| 42 |
+
{{- '<|im_start|>user' }}
|
| 43 |
+
{%- endif %}
|
| 44 |
+
{{- '\n<tool_response>\n' }}
|
| 45 |
+
{{- message.content }}
|
| 46 |
+
{{- '\n</tool_response>' }}
|
| 47 |
+
{%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
|
| 48 |
+
{{- '<|im_end|>\n' }}
|
| 49 |
+
{%- endif %}
|
| 50 |
+
{%- endif %}
|
| 51 |
+
{%- endfor %}
|
| 52 |
+
{%- if add_generation_prompt %}
|
| 53 |
+
{{- '<|im_start|>assistant\n' }}
|
| 54 |
+
{%- endif %}
|
config.json
ADDED
|
@@ -0,0 +1,169 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"Audio8ASRInfiniteForConditionalGeneration"
|
| 4 |
+
],
|
| 5 |
+
"audio_config": {
|
| 6 |
+
"activation_function": "gelu",
|
| 7 |
+
"attention_dropout": 0.0,
|
| 8 |
+
"dtype": "bfloat16",
|
| 9 |
+
"head_dim": 64,
|
| 10 |
+
"hidden_act": "silu",
|
| 11 |
+
"hidden_size": 1280,
|
| 12 |
+
"initializer_range": 0.02,
|
| 13 |
+
"intermediate_size": 5120,
|
| 14 |
+
"max_position_embeddings": 1500,
|
| 15 |
+
"model_type": "voxtral_realtime_encoder",
|
| 16 |
+
"num_attention_heads": 32,
|
| 17 |
+
"num_hidden_layers": 32,
|
| 18 |
+
"num_mel_bins": 128,
|
| 19 |
+
"rms_norm_eps": 1e-05,
|
| 20 |
+
"rope_parameters": {
|
| 21 |
+
"rope_theta": 1000000.0,
|
| 22 |
+
"rope_type": "default"
|
| 23 |
+
},
|
| 24 |
+
"sliding_window": 750,
|
| 25 |
+
"streaming_n_left_pad_tokens": 18,
|
| 26 |
+
"vocab_size": 131072
|
| 27 |
+
},
|
| 28 |
+
"audio_length_per_tok": 8,
|
| 29 |
+
"audio_tower_frame_ms": 20,
|
| 30 |
+
"auto_map": {
|
| 31 |
+
"AutoConfig": "configuration_audio8_asr_infinite.Audio8ASRInfiniteConfig",
|
| 32 |
+
"AutoModelForCausalLM": "modeling_audio8_asr_infinite.Audio8ASRInfiniteForConditionalGeneration"
|
| 33 |
+
},
|
| 34 |
+
"bos_token_id": 151644,
|
| 35 |
+
"default_num_delay_tokens": null,
|
| 36 |
+
"dtype": "bfloat16",
|
| 37 |
+
"eos_token_id": 151645,
|
| 38 |
+
"frame_lens": [
|
| 39 |
+
4,
|
| 40 |
+
6,
|
| 41 |
+
8
|
| 42 |
+
],
|
| 43 |
+
"hidden_size": 2048,
|
| 44 |
+
"max_frame_len": 8,
|
| 45 |
+
"model_type": "audio8_asr_infinite",
|
| 46 |
+
"num_delay_tokens_by_frame_len": {
|
| 47 |
+
"4": {
|
| 48 |
+
"240": 3,
|
| 49 |
+
"320": 4,
|
| 50 |
+
"480": 6,
|
| 51 |
+
"560": 7
|
| 52 |
+
},
|
| 53 |
+
"6": {
|
| 54 |
+
"240": 2,
|
| 55 |
+
"480": 4
|
| 56 |
+
},
|
| 57 |
+
"8": {
|
| 58 |
+
"320": 2,
|
| 59 |
+
"480": 3
|
| 60 |
+
}
|
| 61 |
+
},
|
| 62 |
+
"pad_token_id": 151643,
|
| 63 |
+
"processor_class": "VoxtralRealtimeFeatureExtractor",
|
| 64 |
+
"projection_size": 10240,
|
| 65 |
+
"projector_hidden_act": "gelu",
|
| 66 |
+
"semantic_vad_horizons_seconds": [
|
| 67 |
+
0.5,
|
| 68 |
+
1.0,
|
| 69 |
+
2.0,
|
| 70 |
+
3.0
|
| 71 |
+
],
|
| 72 |
+
"semantic_vad_num_classes": 8,
|
| 73 |
+
"streaming_frame_ms": null,
|
| 74 |
+
"streaming_n_left_pad_tokens": 18,
|
| 75 |
+
"streaming_n_left_pad_tokens_by_frame_len": {
|
| 76 |
+
"4": 18,
|
| 77 |
+
"6": 12,
|
| 78 |
+
"8": 9
|
| 79 |
+
},
|
| 80 |
+
"supported_frame_lens": [
|
| 81 |
+
4,
|
| 82 |
+
6,
|
| 83 |
+
8
|
| 84 |
+
],
|
| 85 |
+
"target_delay_ms": [
|
| 86 |
+
240,
|
| 87 |
+
320,
|
| 88 |
+
480,
|
| 89 |
+
560
|
| 90 |
+
],
|
| 91 |
+
"text_config": {
|
| 92 |
+
"_name_or_path": "Qwen/Qwen2.5-3B-Instruct",
|
| 93 |
+
"architectures": [
|
| 94 |
+
"Qwen2ForCausalLM"
|
| 95 |
+
],
|
| 96 |
+
"attention_dropout": 0.0,
|
| 97 |
+
"bos_token_id": 151644,
|
| 98 |
+
"dtype": "bfloat16",
|
| 99 |
+
"eos_token_id": 151645,
|
| 100 |
+
"hidden_act": "silu",
|
| 101 |
+
"hidden_size": 2048,
|
| 102 |
+
"initializer_range": 0.02,
|
| 103 |
+
"intermediate_size": 11008,
|
| 104 |
+
"layer_types": [
|
| 105 |
+
"full_attention",
|
| 106 |
+
"full_attention",
|
| 107 |
+
"full_attention",
|
| 108 |
+
"full_attention",
|
| 109 |
+
"full_attention",
|
| 110 |
+
"full_attention",
|
| 111 |
+
"full_attention",
|
| 112 |
+
"full_attention",
|
| 113 |
+
"full_attention",
|
| 114 |
+
"full_attention",
|
| 115 |
+
"full_attention",
|
| 116 |
+
"full_attention",
|
| 117 |
+
"full_attention",
|
| 118 |
+
"full_attention",
|
| 119 |
+
"full_attention",
|
| 120 |
+
"full_attention",
|
| 121 |
+
"full_attention",
|
| 122 |
+
"full_attention",
|
| 123 |
+
"full_attention",
|
| 124 |
+
"full_attention",
|
| 125 |
+
"full_attention",
|
| 126 |
+
"full_attention",
|
| 127 |
+
"full_attention",
|
| 128 |
+
"full_attention",
|
| 129 |
+
"full_attention",
|
| 130 |
+
"full_attention",
|
| 131 |
+
"full_attention",
|
| 132 |
+
"full_attention",
|
| 133 |
+
"full_attention",
|
| 134 |
+
"full_attention",
|
| 135 |
+
"full_attention",
|
| 136 |
+
"full_attention",
|
| 137 |
+
"full_attention",
|
| 138 |
+
"full_attention",
|
| 139 |
+
"full_attention",
|
| 140 |
+
"full_attention"
|
| 141 |
+
],
|
| 142 |
+
"max_position_embeddings": 32768,
|
| 143 |
+
"max_window_layers": 70,
|
| 144 |
+
"model_type": "qwen2",
|
| 145 |
+
"num_attention_heads": 16,
|
| 146 |
+
"num_hidden_layers": 36,
|
| 147 |
+
"num_key_value_heads": 2,
|
| 148 |
+
"pad_token_id": 151643,
|
| 149 |
+
"projection_size": 10240,
|
| 150 |
+
"rms_norm_eps": 1e-06,
|
| 151 |
+
"rope_parameters": {
|
| 152 |
+
"rope_theta": 1000000.0,
|
| 153 |
+
"rope_type": "default"
|
| 154 |
+
},
|
| 155 |
+
"sliding_window": null,
|
| 156 |
+
"tie_word_embeddings": true,
|
| 157 |
+
"use_cache": false,
|
| 158 |
+
"use_sliding_window": false,
|
| 159 |
+
"vocab_size": 151936
|
| 160 |
+
},
|
| 161 |
+
"tie_word_embeddings": true,
|
| 162 |
+
"token_duration_ms": null,
|
| 163 |
+
"tokenizer_class": "Qwen2Tokenizer",
|
| 164 |
+
"transformers_version": "5.13.0",
|
| 165 |
+
"use_cache": false,
|
| 166 |
+
"use_frame_len_embedding": true,
|
| 167 |
+
"vocab_size": 151936,
|
| 168 |
+
"weight_format_version": 2
|
| 169 |
+
}
|
configuration_audio8_asr_infinite.py
ADDED
|
@@ -0,0 +1,230 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Configuration for Audio8 ASR Infinite."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from typing import Any, Mapping, Sequence
|
| 6 |
+
|
| 7 |
+
from transformers import PretrainedConfig, Qwen2Config, Qwen3Config
|
| 8 |
+
from transformers.models.voxtral_realtime.configuration_voxtral_realtime import (
|
| 9 |
+
VoxtralRealtimeEncoderConfig,
|
| 10 |
+
)
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
AUDIO8_ASR_INFINITE_WEIGHT_FORMAT_VERSION = 2
|
| 14 |
+
|
| 15 |
+
# Semantic VAD head contract: one classifier per future horizon, each predicting
|
| 16 |
+
# the number of semantic units (0..num_classes-1) that will appear within that
|
| 17 |
+
# horizon. Class 0 is the end-of-turn class the realtime client thresholds on.
|
| 18 |
+
DEFAULT_SEMANTIC_VAD_NUM_CLASSES = 8
|
| 19 |
+
DEFAULT_SEMANTIC_VAD_HORIZONS_SECONDS: tuple[float, ...] = (0.5, 1.0, 2.0, 3.0)
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class Audio8ASRInfiniteConfig(PretrainedConfig):
|
| 23 |
+
"""Configuration for the Audio8 ASR Infinite model."""
|
| 24 |
+
|
| 25 |
+
model_type = "audio8_asr_infinite"
|
| 26 |
+
sub_configs = {
|
| 27 |
+
"audio_config": VoxtralRealtimeEncoderConfig,
|
| 28 |
+
"text_config": Qwen3Config,
|
| 29 |
+
}
|
| 30 |
+
|
| 31 |
+
def __init__(
|
| 32 |
+
self,
|
| 33 |
+
audio_config: Mapping[str, Any] | VoxtralRealtimeEncoderConfig | None = None,
|
| 34 |
+
text_config: Mapping[str, Any] | Qwen2Config | Qwen3Config | None = None,
|
| 35 |
+
audio_length_per_tok: int = 8,
|
| 36 |
+
default_num_delay_tokens: int | None = None,
|
| 37 |
+
supported_frame_lens: Sequence[int] = (4, 6, 8),
|
| 38 |
+
audio_tower_frame_ms: int = 20,
|
| 39 |
+
use_frame_len_embedding: bool = False,
|
| 40 |
+
projector_hidden_act: str = "gelu",
|
| 41 |
+
semantic_vad_horizons_seconds: Sequence[float] | None = None,
|
| 42 |
+
semantic_vad_num_classes: int = DEFAULT_SEMANTIC_VAD_NUM_CLASSES,
|
| 43 |
+
weight_format_version: int = AUDIO8_ASR_INFINITE_WEIGHT_FORMAT_VERSION,
|
| 44 |
+
**kwargs: Any,
|
| 45 |
+
) -> None:
|
| 46 |
+
super().__init__(**kwargs)
|
| 47 |
+
if audio_config is None:
|
| 48 |
+
audio_config = VoxtralRealtimeEncoderConfig()
|
| 49 |
+
if isinstance(audio_config, Mapping):
|
| 50 |
+
audio_config = VoxtralRealtimeEncoderConfig(**dict(audio_config))
|
| 51 |
+
if text_config is None:
|
| 52 |
+
text_config = Qwen3Config()
|
| 53 |
+
if isinstance(text_config, Mapping):
|
| 54 |
+
text_config_payload = dict(text_config)
|
| 55 |
+
text_model_type = text_config_payload.pop(
|
| 56 |
+
"model_type",
|
| 57 |
+
Qwen3Config.model_type,
|
| 58 |
+
)
|
| 59 |
+
text_config_class = {
|
| 60 |
+
Qwen2Config.model_type: Qwen2Config,
|
| 61 |
+
Qwen3Config.model_type: Qwen3Config,
|
| 62 |
+
}.get(str(text_model_type))
|
| 63 |
+
if text_config_class is None:
|
| 64 |
+
raise ValueError(
|
| 65 |
+
"Audio8 ASR Infinite text_config must use Qwen2 or Qwen3, "
|
| 66 |
+
f"got model_type={text_model_type!r}."
|
| 67 |
+
)
|
| 68 |
+
text_config = text_config_class(**text_config_payload)
|
| 69 |
+
|
| 70 |
+
self.audio_config = audio_config
|
| 71 |
+
self.text_config = text_config
|
| 72 |
+
self.tie_word_embeddings = bool(text_config.tie_word_embeddings)
|
| 73 |
+
self.audio_length_per_tok = int(audio_length_per_tok)
|
| 74 |
+
self.default_num_delay_tokens = (
|
| 75 |
+
None
|
| 76 |
+
if default_num_delay_tokens is None
|
| 77 |
+
else int(default_num_delay_tokens)
|
| 78 |
+
)
|
| 79 |
+
self.supported_frame_lens = tuple(
|
| 80 |
+
int(value) for value in supported_frame_lens
|
| 81 |
+
)
|
| 82 |
+
self.audio_tower_frame_ms = int(audio_tower_frame_ms)
|
| 83 |
+
if self.audio_tower_frame_ms <= 0:
|
| 84 |
+
raise ValueError("audio_tower_frame_ms must be positive.")
|
| 85 |
+
if (
|
| 86 |
+
not self.supported_frame_lens
|
| 87 |
+
or any(value <= 0 for value in self.supported_frame_lens)
|
| 88 |
+
or len(set(self.supported_frame_lens))
|
| 89 |
+
!= len(self.supported_frame_lens)
|
| 90 |
+
):
|
| 91 |
+
raise ValueError(
|
| 92 |
+
"supported_frame_lens must contain unique positive integers."
|
| 93 |
+
)
|
| 94 |
+
self.max_frame_len = max(self.supported_frame_lens)
|
| 95 |
+
self.use_frame_len_embedding = bool(
|
| 96 |
+
use_frame_len_embedding
|
| 97 |
+
and len(self.supported_frame_lens) > 1
|
| 98 |
+
)
|
| 99 |
+
self.projector_hidden_act = str(projector_hidden_act)
|
| 100 |
+
# `None` / empty means "no semantic VAD heads": plain transcription
|
| 101 |
+
# checkpoints are unaffected and their weight keys are unchanged.
|
| 102 |
+
self.semantic_vad_horizons_seconds = (
|
| 103 |
+
None
|
| 104 |
+
if semantic_vad_horizons_seconds is None
|
| 105 |
+
else [float(horizon) for horizon in semantic_vad_horizons_seconds]
|
| 106 |
+
)
|
| 107 |
+
if self.semantic_vad_horizons_seconds is not None and (
|
| 108 |
+
not self.semantic_vad_horizons_seconds
|
| 109 |
+
or any(
|
| 110 |
+
horizon <= 0.0 for horizon in self.semantic_vad_horizons_seconds
|
| 111 |
+
)
|
| 112 |
+
):
|
| 113 |
+
raise ValueError(
|
| 114 |
+
"semantic_vad_horizons_seconds must be a non-empty sequence of "
|
| 115 |
+
"positive numbers, or None for a transcription-only checkpoint."
|
| 116 |
+
)
|
| 117 |
+
self.semantic_vad_num_classes = int(semantic_vad_num_classes)
|
| 118 |
+
if self.semantic_vad_num_classes < 2:
|
| 119 |
+
raise ValueError("semantic_vad_num_classes must be at least 2.")
|
| 120 |
+
self.weight_format_version = int(weight_format_version)
|
| 121 |
+
if (
|
| 122 |
+
self.weight_format_version
|
| 123 |
+
!= AUDIO8_ASR_INFINITE_WEIGHT_FORMAT_VERSION
|
| 124 |
+
):
|
| 125 |
+
raise ValueError(
|
| 126 |
+
"Unsupported Audio8 ASR Infinite weight format: "
|
| 127 |
+
f"expected={AUDIO8_ASR_INFINITE_WEIGHT_FORMAT_VERSION} "
|
| 128 |
+
f"got={self.weight_format_version}. Convert the checkpoint "
|
| 129 |
+
"before loading it."
|
| 130 |
+
)
|
| 131 |
+
self.projection_size = (
|
| 132 |
+
int(self.audio_config.hidden_size)
|
| 133 |
+
* self.max_frame_len
|
| 134 |
+
)
|
| 135 |
+
self.text_config.projection_size = self.projection_size
|
| 136 |
+
self.vocab_size = int(text_config.vocab_size)
|
| 137 |
+
self.hidden_size = int(text_config.hidden_size)
|
| 138 |
+
self.pad_token_id = text_config.pad_token_id
|
| 139 |
+
self.bos_token_id = text_config.bos_token_id
|
| 140 |
+
self.eos_token_id = text_config.eos_token_id
|
| 141 |
+
|
| 142 |
+
@classmethod
|
| 143 |
+
def from_dict(
|
| 144 |
+
cls,
|
| 145 |
+
config_dict: dict[str, Any],
|
| 146 |
+
**kwargs: Any,
|
| 147 |
+
) -> Audio8ASRInfiniteConfig:
|
| 148 |
+
if config_dict.get("model_type") != cls.model_type:
|
| 149 |
+
raise ValueError(
|
| 150 |
+
"Audio8 ASR Infinite only loads its own checkpoint format: "
|
| 151 |
+
f"expected model_type={cls.model_type!r}, got "
|
| 152 |
+
f"{config_dict.get('model_type')!r}. Convert the checkpoint "
|
| 153 |
+
"before loading it."
|
| 154 |
+
)
|
| 155 |
+
if "weight_format_version" not in config_dict:
|
| 156 |
+
raise ValueError(
|
| 157 |
+
"Audio8 ASR Infinite checkpoint is missing "
|
| 158 |
+
"`weight_format_version`. Convert the checkpoint to the "
|
| 159 |
+
"current format before loading it."
|
| 160 |
+
)
|
| 161 |
+
return super().from_dict(config_dict, **kwargs)
|
| 162 |
+
|
| 163 |
+
def to_dict(self) -> dict[str, Any]:
|
| 164 |
+
output = super().to_dict()
|
| 165 |
+
output["audio_config"] = self.audio_config.to_dict()
|
| 166 |
+
output["text_config"] = self.text_config.to_dict()
|
| 167 |
+
output["audio_length_per_tok"] = self.audio_length_per_tok
|
| 168 |
+
output["default_num_delay_tokens"] = self.default_num_delay_tokens
|
| 169 |
+
output["supported_frame_lens"] = list(
|
| 170 |
+
self.supported_frame_lens
|
| 171 |
+
)
|
| 172 |
+
output["max_frame_len"] = self.max_frame_len
|
| 173 |
+
output["audio_tower_frame_ms"] = self.audio_tower_frame_ms
|
| 174 |
+
output["use_frame_len_embedding"] = (
|
| 175 |
+
self.use_frame_len_embedding
|
| 176 |
+
)
|
| 177 |
+
output["projector_hidden_act"] = self.projector_hidden_act
|
| 178 |
+
output["weight_format_version"] = self.weight_format_version
|
| 179 |
+
output["projection_size"] = self.projection_size
|
| 180 |
+
output["text_config"]["projection_size"] = self.projection_size
|
| 181 |
+
output["model_type"] = self.model_type
|
| 182 |
+
return output
|
| 183 |
+
|
| 184 |
+
def resolve_frame_len(
|
| 185 |
+
self,
|
| 186 |
+
streaming_frame_ms: int,
|
| 187 |
+
) -> int:
|
| 188 |
+
streaming_frame_ms = int(streaming_frame_ms)
|
| 189 |
+
if streaming_frame_ms <= 0:
|
| 190 |
+
raise ValueError("streaming_frame_ms must be positive.")
|
| 191 |
+
if streaming_frame_ms % self.audio_tower_frame_ms != 0:
|
| 192 |
+
raise ValueError(
|
| 193 |
+
"streaming_frame_ms must be divisible by "
|
| 194 |
+
f"audio_tower_frame_ms={self.audio_tower_frame_ms}, got "
|
| 195 |
+
f"{streaming_frame_ms}."
|
| 196 |
+
)
|
| 197 |
+
frame_len = (
|
| 198 |
+
streaming_frame_ms // self.audio_tower_frame_ms
|
| 199 |
+
)
|
| 200 |
+
if frame_len not in self.supported_frame_lens:
|
| 201 |
+
raise ValueError(
|
| 202 |
+
"streaming_frame_ms resolves to unsupported "
|
| 203 |
+
f"frame_len={frame_len}; supported="
|
| 204 |
+
f"{self.supported_frame_lens}."
|
| 205 |
+
)
|
| 206 |
+
return frame_len
|
| 207 |
+
|
| 208 |
+
def resolve_num_delay_tokens(
|
| 209 |
+
self,
|
| 210 |
+
*,
|
| 211 |
+
target_delay_ms: int,
|
| 212 |
+
streaming_frame_ms: int,
|
| 213 |
+
) -> int:
|
| 214 |
+
self.resolve_frame_len(streaming_frame_ms)
|
| 215 |
+
target_delay_ms = int(target_delay_ms)
|
| 216 |
+
if target_delay_ms <= 0:
|
| 217 |
+
raise ValueError("target_delay_ms must be positive.")
|
| 218 |
+
if target_delay_ms % int(streaming_frame_ms) != 0:
|
| 219 |
+
raise ValueError(
|
| 220 |
+
"target_delay_ms must be divisible by streaming_frame_ms, "
|
| 221 |
+
f"got target_delay_ms={target_delay_ms} "
|
| 222 |
+
f"streaming_frame_ms={streaming_frame_ms}."
|
| 223 |
+
)
|
| 224 |
+
return target_delay_ms // int(streaming_frame_ms)
|
| 225 |
+
|
| 226 |
+
|
| 227 |
+
__all__ = [
|
| 228 |
+
"AUDIO8_ASR_INFINITE_WEIGHT_FORMAT_VERSION",
|
| 229 |
+
"Audio8ASRInfiniteConfig",
|
| 230 |
+
]
|
generation_config.json
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_from_model_config": true,
|
| 3 |
+
"bos_token_id": 151644,
|
| 4 |
+
"eos_token_id": [
|
| 5 |
+
151645
|
| 6 |
+
],
|
| 7 |
+
"output_attentions": false,
|
| 8 |
+
"output_hidden_states": false,
|
| 9 |
+
"pad_token_id": 151643,
|
| 10 |
+
"transformers_version": "5.13.0",
|
| 11 |
+
"use_cache": true
|
| 12 |
+
}
|
model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2cf97d69e9f5853855b783b359dffb661c035281a02053782c21fe009e31c7ba
|
| 3 |
+
size 8172565584
|
model.safetensors.index.json
ADDED
|
@@ -0,0 +1,945 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"metadata": {
|
| 3 |
+
"total_size": 8172580416
|
| 4 |
+
},
|
| 5 |
+
"weight_map": {
|
| 6 |
+
"audio_tower.embedder.conv1.bias": "model.safetensors",
|
| 7 |
+
"audio_tower.embedder.conv1.weight": "model.safetensors",
|
| 8 |
+
"audio_tower.embedder.conv2.bias": "model.safetensors",
|
| 9 |
+
"audio_tower.embedder.conv2.weight": "model.safetensors",
|
| 10 |
+
"audio_tower.layers.0.final_layer_norm.weight": "model.safetensors",
|
| 11 |
+
"audio_tower.layers.0.mlp.down_proj.bias": "model.safetensors",
|
| 12 |
+
"audio_tower.layers.0.mlp.down_proj.weight": "model.safetensors",
|
| 13 |
+
"audio_tower.layers.0.mlp.gate_proj.weight": "model.safetensors",
|
| 14 |
+
"audio_tower.layers.0.mlp.up_proj.weight": "model.safetensors",
|
| 15 |
+
"audio_tower.layers.0.self_attn.k_proj.weight": "model.safetensors",
|
| 16 |
+
"audio_tower.layers.0.self_attn.o_proj.bias": "model.safetensors",
|
| 17 |
+
"audio_tower.layers.0.self_attn.o_proj.weight": "model.safetensors",
|
| 18 |
+
"audio_tower.layers.0.self_attn.q_proj.bias": "model.safetensors",
|
| 19 |
+
"audio_tower.layers.0.self_attn.q_proj.weight": "model.safetensors",
|
| 20 |
+
"audio_tower.layers.0.self_attn.v_proj.bias": "model.safetensors",
|
| 21 |
+
"audio_tower.layers.0.self_attn.v_proj.weight": "model.safetensors",
|
| 22 |
+
"audio_tower.layers.0.self_attn_layer_norm.weight": "model.safetensors",
|
| 23 |
+
"audio_tower.layers.1.final_layer_norm.weight": "model.safetensors",
|
| 24 |
+
"audio_tower.layers.1.mlp.down_proj.bias": "model.safetensors",
|
| 25 |
+
"audio_tower.layers.1.mlp.down_proj.weight": "model.safetensors",
|
| 26 |
+
"audio_tower.layers.1.mlp.gate_proj.weight": "model.safetensors",
|
| 27 |
+
"audio_tower.layers.1.mlp.up_proj.weight": "model.safetensors",
|
| 28 |
+
"audio_tower.layers.1.self_attn.k_proj.weight": "model.safetensors",
|
| 29 |
+
"audio_tower.layers.1.self_attn.o_proj.bias": "model.safetensors",
|
| 30 |
+
"audio_tower.layers.1.self_attn.o_proj.weight": "model.safetensors",
|
| 31 |
+
"audio_tower.layers.1.self_attn.q_proj.bias": "model.safetensors",
|
| 32 |
+
"audio_tower.layers.1.self_attn.q_proj.weight": "model.safetensors",
|
| 33 |
+
"audio_tower.layers.1.self_attn.v_proj.bias": "model.safetensors",
|
| 34 |
+
"audio_tower.layers.1.self_attn.v_proj.weight": "model.safetensors",
|
| 35 |
+
"audio_tower.layers.1.self_attn_layer_norm.weight": "model.safetensors",
|
| 36 |
+
"audio_tower.layers.10.final_layer_norm.weight": "model.safetensors",
|
| 37 |
+
"audio_tower.layers.10.mlp.down_proj.bias": "model.safetensors",
|
| 38 |
+
"audio_tower.layers.10.mlp.down_proj.weight": "model.safetensors",
|
| 39 |
+
"audio_tower.layers.10.mlp.gate_proj.weight": "model.safetensors",
|
| 40 |
+
"audio_tower.layers.10.mlp.up_proj.weight": "model.safetensors",
|
| 41 |
+
"audio_tower.layers.10.self_attn.k_proj.weight": "model.safetensors",
|
| 42 |
+
"audio_tower.layers.10.self_attn.o_proj.bias": "model.safetensors",
|
| 43 |
+
"audio_tower.layers.10.self_attn.o_proj.weight": "model.safetensors",
|
| 44 |
+
"audio_tower.layers.10.self_attn.q_proj.bias": "model.safetensors",
|
| 45 |
+
"audio_tower.layers.10.self_attn.q_proj.weight": "model.safetensors",
|
| 46 |
+
"audio_tower.layers.10.self_attn.v_proj.bias": "model.safetensors",
|
| 47 |
+
"audio_tower.layers.10.self_attn.v_proj.weight": "model.safetensors",
|
| 48 |
+
"audio_tower.layers.10.self_attn_layer_norm.weight": "model.safetensors",
|
| 49 |
+
"audio_tower.layers.11.final_layer_norm.weight": "model.safetensors",
|
| 50 |
+
"audio_tower.layers.11.mlp.down_proj.bias": "model.safetensors",
|
| 51 |
+
"audio_tower.layers.11.mlp.down_proj.weight": "model.safetensors",
|
| 52 |
+
"audio_tower.layers.11.mlp.gate_proj.weight": "model.safetensors",
|
| 53 |
+
"audio_tower.layers.11.mlp.up_proj.weight": "model.safetensors",
|
| 54 |
+
"audio_tower.layers.11.self_attn.k_proj.weight": "model.safetensors",
|
| 55 |
+
"audio_tower.layers.11.self_attn.o_proj.bias": "model.safetensors",
|
| 56 |
+
"audio_tower.layers.11.self_attn.o_proj.weight": "model.safetensors",
|
| 57 |
+
"audio_tower.layers.11.self_attn.q_proj.bias": "model.safetensors",
|
| 58 |
+
"audio_tower.layers.11.self_attn.q_proj.weight": "model.safetensors",
|
| 59 |
+
"audio_tower.layers.11.self_attn.v_proj.bias": "model.safetensors",
|
| 60 |
+
"audio_tower.layers.11.self_attn.v_proj.weight": "model.safetensors",
|
| 61 |
+
"audio_tower.layers.11.self_attn_layer_norm.weight": "model.safetensors",
|
| 62 |
+
"audio_tower.layers.12.final_layer_norm.weight": "model.safetensors",
|
| 63 |
+
"audio_tower.layers.12.mlp.down_proj.bias": "model.safetensors",
|
| 64 |
+
"audio_tower.layers.12.mlp.down_proj.weight": "model.safetensors",
|
| 65 |
+
"audio_tower.layers.12.mlp.gate_proj.weight": "model.safetensors",
|
| 66 |
+
"audio_tower.layers.12.mlp.up_proj.weight": "model.safetensors",
|
| 67 |
+
"audio_tower.layers.12.self_attn.k_proj.weight": "model.safetensors",
|
| 68 |
+
"audio_tower.layers.12.self_attn.o_proj.bias": "model.safetensors",
|
| 69 |
+
"audio_tower.layers.12.self_attn.o_proj.weight": "model.safetensors",
|
| 70 |
+
"audio_tower.layers.12.self_attn.q_proj.bias": "model.safetensors",
|
| 71 |
+
"audio_tower.layers.12.self_attn.q_proj.weight": "model.safetensors",
|
| 72 |
+
"audio_tower.layers.12.self_attn.v_proj.bias": "model.safetensors",
|
| 73 |
+
"audio_tower.layers.12.self_attn.v_proj.weight": "model.safetensors",
|
| 74 |
+
"audio_tower.layers.12.self_attn_layer_norm.weight": "model.safetensors",
|
| 75 |
+
"audio_tower.layers.13.final_layer_norm.weight": "model.safetensors",
|
| 76 |
+
"audio_tower.layers.13.mlp.down_proj.bias": "model.safetensors",
|
| 77 |
+
"audio_tower.layers.13.mlp.down_proj.weight": "model.safetensors",
|
| 78 |
+
"audio_tower.layers.13.mlp.gate_proj.weight": "model.safetensors",
|
| 79 |
+
"audio_tower.layers.13.mlp.up_proj.weight": "model.safetensors",
|
| 80 |
+
"audio_tower.layers.13.self_attn.k_proj.weight": "model.safetensors",
|
| 81 |
+
"audio_tower.layers.13.self_attn.o_proj.bias": "model.safetensors",
|
| 82 |
+
"audio_tower.layers.13.self_attn.o_proj.weight": "model.safetensors",
|
| 83 |
+
"audio_tower.layers.13.self_attn.q_proj.bias": "model.safetensors",
|
| 84 |
+
"audio_tower.layers.13.self_attn.q_proj.weight": "model.safetensors",
|
| 85 |
+
"audio_tower.layers.13.self_attn.v_proj.bias": "model.safetensors",
|
| 86 |
+
"audio_tower.layers.13.self_attn.v_proj.weight": "model.safetensors",
|
| 87 |
+
"audio_tower.layers.13.self_attn_layer_norm.weight": "model.safetensors",
|
| 88 |
+
"audio_tower.layers.14.final_layer_norm.weight": "model.safetensors",
|
| 89 |
+
"audio_tower.layers.14.mlp.down_proj.bias": "model.safetensors",
|
| 90 |
+
"audio_tower.layers.14.mlp.down_proj.weight": "model.safetensors",
|
| 91 |
+
"audio_tower.layers.14.mlp.gate_proj.weight": "model.safetensors",
|
| 92 |
+
"audio_tower.layers.14.mlp.up_proj.weight": "model.safetensors",
|
| 93 |
+
"audio_tower.layers.14.self_attn.k_proj.weight": "model.safetensors",
|
| 94 |
+
"audio_tower.layers.14.self_attn.o_proj.bias": "model.safetensors",
|
| 95 |
+
"audio_tower.layers.14.self_attn.o_proj.weight": "model.safetensors",
|
| 96 |
+
"audio_tower.layers.14.self_attn.q_proj.bias": "model.safetensors",
|
| 97 |
+
"audio_tower.layers.14.self_attn.q_proj.weight": "model.safetensors",
|
| 98 |
+
"audio_tower.layers.14.self_attn.v_proj.bias": "model.safetensors",
|
| 99 |
+
"audio_tower.layers.14.self_attn.v_proj.weight": "model.safetensors",
|
| 100 |
+
"audio_tower.layers.14.self_attn_layer_norm.weight": "model.safetensors",
|
| 101 |
+
"audio_tower.layers.15.final_layer_norm.weight": "model.safetensors",
|
| 102 |
+
"audio_tower.layers.15.mlp.down_proj.bias": "model.safetensors",
|
| 103 |
+
"audio_tower.layers.15.mlp.down_proj.weight": "model.safetensors",
|
| 104 |
+
"audio_tower.layers.15.mlp.gate_proj.weight": "model.safetensors",
|
| 105 |
+
"audio_tower.layers.15.mlp.up_proj.weight": "model.safetensors",
|
| 106 |
+
"audio_tower.layers.15.self_attn.k_proj.weight": "model.safetensors",
|
| 107 |
+
"audio_tower.layers.15.self_attn.o_proj.bias": "model.safetensors",
|
| 108 |
+
"audio_tower.layers.15.self_attn.o_proj.weight": "model.safetensors",
|
| 109 |
+
"audio_tower.layers.15.self_attn.q_proj.bias": "model.safetensors",
|
| 110 |
+
"audio_tower.layers.15.self_attn.q_proj.weight": "model.safetensors",
|
| 111 |
+
"audio_tower.layers.15.self_attn.v_proj.bias": "model.safetensors",
|
| 112 |
+
"audio_tower.layers.15.self_attn.v_proj.weight": "model.safetensors",
|
| 113 |
+
"audio_tower.layers.15.self_attn_layer_norm.weight": "model.safetensors",
|
| 114 |
+
"audio_tower.layers.16.final_layer_norm.weight": "model.safetensors",
|
| 115 |
+
"audio_tower.layers.16.mlp.down_proj.bias": "model.safetensors",
|
| 116 |
+
"audio_tower.layers.16.mlp.down_proj.weight": "model.safetensors",
|
| 117 |
+
"audio_tower.layers.16.mlp.gate_proj.weight": "model.safetensors",
|
| 118 |
+
"audio_tower.layers.16.mlp.up_proj.weight": "model.safetensors",
|
| 119 |
+
"audio_tower.layers.16.self_attn.k_proj.weight": "model.safetensors",
|
| 120 |
+
"audio_tower.layers.16.self_attn.o_proj.bias": "model.safetensors",
|
| 121 |
+
"audio_tower.layers.16.self_attn.o_proj.weight": "model.safetensors",
|
| 122 |
+
"audio_tower.layers.16.self_attn.q_proj.bias": "model.safetensors",
|
| 123 |
+
"audio_tower.layers.16.self_attn.q_proj.weight": "model.safetensors",
|
| 124 |
+
"audio_tower.layers.16.self_attn.v_proj.bias": "model.safetensors",
|
| 125 |
+
"audio_tower.layers.16.self_attn.v_proj.weight": "model.safetensors",
|
| 126 |
+
"audio_tower.layers.16.self_attn_layer_norm.weight": "model.safetensors",
|
| 127 |
+
"audio_tower.layers.17.final_layer_norm.weight": "model.safetensors",
|
| 128 |
+
"audio_tower.layers.17.mlp.down_proj.bias": "model.safetensors",
|
| 129 |
+
"audio_tower.layers.17.mlp.down_proj.weight": "model.safetensors",
|
| 130 |
+
"audio_tower.layers.17.mlp.gate_proj.weight": "model.safetensors",
|
| 131 |
+
"audio_tower.layers.17.mlp.up_proj.weight": "model.safetensors",
|
| 132 |
+
"audio_tower.layers.17.self_attn.k_proj.weight": "model.safetensors",
|
| 133 |
+
"audio_tower.layers.17.self_attn.o_proj.bias": "model.safetensors",
|
| 134 |
+
"audio_tower.layers.17.self_attn.o_proj.weight": "model.safetensors",
|
| 135 |
+
"audio_tower.layers.17.self_attn.q_proj.bias": "model.safetensors",
|
| 136 |
+
"audio_tower.layers.17.self_attn.q_proj.weight": "model.safetensors",
|
| 137 |
+
"audio_tower.layers.17.self_attn.v_proj.bias": "model.safetensors",
|
| 138 |
+
"audio_tower.layers.17.self_attn.v_proj.weight": "model.safetensors",
|
| 139 |
+
"audio_tower.layers.17.self_attn_layer_norm.weight": "model.safetensors",
|
| 140 |
+
"audio_tower.layers.18.final_layer_norm.weight": "model.safetensors",
|
| 141 |
+
"audio_tower.layers.18.mlp.down_proj.bias": "model.safetensors",
|
| 142 |
+
"audio_tower.layers.18.mlp.down_proj.weight": "model.safetensors",
|
| 143 |
+
"audio_tower.layers.18.mlp.gate_proj.weight": "model.safetensors",
|
| 144 |
+
"audio_tower.layers.18.mlp.up_proj.weight": "model.safetensors",
|
| 145 |
+
"audio_tower.layers.18.self_attn.k_proj.weight": "model.safetensors",
|
| 146 |
+
"audio_tower.layers.18.self_attn.o_proj.bias": "model.safetensors",
|
| 147 |
+
"audio_tower.layers.18.self_attn.o_proj.weight": "model.safetensors",
|
| 148 |
+
"audio_tower.layers.18.self_attn.q_proj.bias": "model.safetensors",
|
| 149 |
+
"audio_tower.layers.18.self_attn.q_proj.weight": "model.safetensors",
|
| 150 |
+
"audio_tower.layers.18.self_attn.v_proj.bias": "model.safetensors",
|
| 151 |
+
"audio_tower.layers.18.self_attn.v_proj.weight": "model.safetensors",
|
| 152 |
+
"audio_tower.layers.18.self_attn_layer_norm.weight": "model.safetensors",
|
| 153 |
+
"audio_tower.layers.19.final_layer_norm.weight": "model.safetensors",
|
| 154 |
+
"audio_tower.layers.19.mlp.down_proj.bias": "model.safetensors",
|
| 155 |
+
"audio_tower.layers.19.mlp.down_proj.weight": "model.safetensors",
|
| 156 |
+
"audio_tower.layers.19.mlp.gate_proj.weight": "model.safetensors",
|
| 157 |
+
"audio_tower.layers.19.mlp.up_proj.weight": "model.safetensors",
|
| 158 |
+
"audio_tower.layers.19.self_attn.k_proj.weight": "model.safetensors",
|
| 159 |
+
"audio_tower.layers.19.self_attn.o_proj.bias": "model.safetensors",
|
| 160 |
+
"audio_tower.layers.19.self_attn.o_proj.weight": "model.safetensors",
|
| 161 |
+
"audio_tower.layers.19.self_attn.q_proj.bias": "model.safetensors",
|
| 162 |
+
"audio_tower.layers.19.self_attn.q_proj.weight": "model.safetensors",
|
| 163 |
+
"audio_tower.layers.19.self_attn.v_proj.bias": "model.safetensors",
|
| 164 |
+
"audio_tower.layers.19.self_attn.v_proj.weight": "model.safetensors",
|
| 165 |
+
"audio_tower.layers.19.self_attn_layer_norm.weight": "model.safetensors",
|
| 166 |
+
"audio_tower.layers.2.final_layer_norm.weight": "model.safetensors",
|
| 167 |
+
"audio_tower.layers.2.mlp.down_proj.bias": "model.safetensors",
|
| 168 |
+
"audio_tower.layers.2.mlp.down_proj.weight": "model.safetensors",
|
| 169 |
+
"audio_tower.layers.2.mlp.gate_proj.weight": "model.safetensors",
|
| 170 |
+
"audio_tower.layers.2.mlp.up_proj.weight": "model.safetensors",
|
| 171 |
+
"audio_tower.layers.2.self_attn.k_proj.weight": "model.safetensors",
|
| 172 |
+
"audio_tower.layers.2.self_attn.o_proj.bias": "model.safetensors",
|
| 173 |
+
"audio_tower.layers.2.self_attn.o_proj.weight": "model.safetensors",
|
| 174 |
+
"audio_tower.layers.2.self_attn.q_proj.bias": "model.safetensors",
|
| 175 |
+
"audio_tower.layers.2.self_attn.q_proj.weight": "model.safetensors",
|
| 176 |
+
"audio_tower.layers.2.self_attn.v_proj.bias": "model.safetensors",
|
| 177 |
+
"audio_tower.layers.2.self_attn.v_proj.weight": "model.safetensors",
|
| 178 |
+
"audio_tower.layers.2.self_attn_layer_norm.weight": "model.safetensors",
|
| 179 |
+
"audio_tower.layers.20.final_layer_norm.weight": "model.safetensors",
|
| 180 |
+
"audio_tower.layers.20.mlp.down_proj.bias": "model.safetensors",
|
| 181 |
+
"audio_tower.layers.20.mlp.down_proj.weight": "model.safetensors",
|
| 182 |
+
"audio_tower.layers.20.mlp.gate_proj.weight": "model.safetensors",
|
| 183 |
+
"audio_tower.layers.20.mlp.up_proj.weight": "model.safetensors",
|
| 184 |
+
"audio_tower.layers.20.self_attn.k_proj.weight": "model.safetensors",
|
| 185 |
+
"audio_tower.layers.20.self_attn.o_proj.bias": "model.safetensors",
|
| 186 |
+
"audio_tower.layers.20.self_attn.o_proj.weight": "model.safetensors",
|
| 187 |
+
"audio_tower.layers.20.self_attn.q_proj.bias": "model.safetensors",
|
| 188 |
+
"audio_tower.layers.20.self_attn.q_proj.weight": "model.safetensors",
|
| 189 |
+
"audio_tower.layers.20.self_attn.v_proj.bias": "model.safetensors",
|
| 190 |
+
"audio_tower.layers.20.self_attn.v_proj.weight": "model.safetensors",
|
| 191 |
+
"audio_tower.layers.20.self_attn_layer_norm.weight": "model.safetensors",
|
| 192 |
+
"audio_tower.layers.21.final_layer_norm.weight": "model.safetensors",
|
| 193 |
+
"audio_tower.layers.21.mlp.down_proj.bias": "model.safetensors",
|
| 194 |
+
"audio_tower.layers.21.mlp.down_proj.weight": "model.safetensors",
|
| 195 |
+
"audio_tower.layers.21.mlp.gate_proj.weight": "model.safetensors",
|
| 196 |
+
"audio_tower.layers.21.mlp.up_proj.weight": "model.safetensors",
|
| 197 |
+
"audio_tower.layers.21.self_attn.k_proj.weight": "model.safetensors",
|
| 198 |
+
"audio_tower.layers.21.self_attn.o_proj.bias": "model.safetensors",
|
| 199 |
+
"audio_tower.layers.21.self_attn.o_proj.weight": "model.safetensors",
|
| 200 |
+
"audio_tower.layers.21.self_attn.q_proj.bias": "model.safetensors",
|
| 201 |
+
"audio_tower.layers.21.self_attn.q_proj.weight": "model.safetensors",
|
| 202 |
+
"audio_tower.layers.21.self_attn.v_proj.bias": "model.safetensors",
|
| 203 |
+
"audio_tower.layers.21.self_attn.v_proj.weight": "model.safetensors",
|
| 204 |
+
"audio_tower.layers.21.self_attn_layer_norm.weight": "model.safetensors",
|
| 205 |
+
"audio_tower.layers.22.final_layer_norm.weight": "model.safetensors",
|
| 206 |
+
"audio_tower.layers.22.mlp.down_proj.bias": "model.safetensors",
|
| 207 |
+
"audio_tower.layers.22.mlp.down_proj.weight": "model.safetensors",
|
| 208 |
+
"audio_tower.layers.22.mlp.gate_proj.weight": "model.safetensors",
|
| 209 |
+
"audio_tower.layers.22.mlp.up_proj.weight": "model.safetensors",
|
| 210 |
+
"audio_tower.layers.22.self_attn.k_proj.weight": "model.safetensors",
|
| 211 |
+
"audio_tower.layers.22.self_attn.o_proj.bias": "model.safetensors",
|
| 212 |
+
"audio_tower.layers.22.self_attn.o_proj.weight": "model.safetensors",
|
| 213 |
+
"audio_tower.layers.22.self_attn.q_proj.bias": "model.safetensors",
|
| 214 |
+
"audio_tower.layers.22.self_attn.q_proj.weight": "model.safetensors",
|
| 215 |
+
"audio_tower.layers.22.self_attn.v_proj.bias": "model.safetensors",
|
| 216 |
+
"audio_tower.layers.22.self_attn.v_proj.weight": "model.safetensors",
|
| 217 |
+
"audio_tower.layers.22.self_attn_layer_norm.weight": "model.safetensors",
|
| 218 |
+
"audio_tower.layers.23.final_layer_norm.weight": "model.safetensors",
|
| 219 |
+
"audio_tower.layers.23.mlp.down_proj.bias": "model.safetensors",
|
| 220 |
+
"audio_tower.layers.23.mlp.down_proj.weight": "model.safetensors",
|
| 221 |
+
"audio_tower.layers.23.mlp.gate_proj.weight": "model.safetensors",
|
| 222 |
+
"audio_tower.layers.23.mlp.up_proj.weight": "model.safetensors",
|
| 223 |
+
"audio_tower.layers.23.self_attn.k_proj.weight": "model.safetensors",
|
| 224 |
+
"audio_tower.layers.23.self_attn.o_proj.bias": "model.safetensors",
|
| 225 |
+
"audio_tower.layers.23.self_attn.o_proj.weight": "model.safetensors",
|
| 226 |
+
"audio_tower.layers.23.self_attn.q_proj.bias": "model.safetensors",
|
| 227 |
+
"audio_tower.layers.23.self_attn.q_proj.weight": "model.safetensors",
|
| 228 |
+
"audio_tower.layers.23.self_attn.v_proj.bias": "model.safetensors",
|
| 229 |
+
"audio_tower.layers.23.self_attn.v_proj.weight": "model.safetensors",
|
| 230 |
+
"audio_tower.layers.23.self_attn_layer_norm.weight": "model.safetensors",
|
| 231 |
+
"audio_tower.layers.24.final_layer_norm.weight": "model.safetensors",
|
| 232 |
+
"audio_tower.layers.24.mlp.down_proj.bias": "model.safetensors",
|
| 233 |
+
"audio_tower.layers.24.mlp.down_proj.weight": "model.safetensors",
|
| 234 |
+
"audio_tower.layers.24.mlp.gate_proj.weight": "model.safetensors",
|
| 235 |
+
"audio_tower.layers.24.mlp.up_proj.weight": "model.safetensors",
|
| 236 |
+
"audio_tower.layers.24.self_attn.k_proj.weight": "model.safetensors",
|
| 237 |
+
"audio_tower.layers.24.self_attn.o_proj.bias": "model.safetensors",
|
| 238 |
+
"audio_tower.layers.24.self_attn.o_proj.weight": "model.safetensors",
|
| 239 |
+
"audio_tower.layers.24.self_attn.q_proj.bias": "model.safetensors",
|
| 240 |
+
"audio_tower.layers.24.self_attn.q_proj.weight": "model.safetensors",
|
| 241 |
+
"audio_tower.layers.24.self_attn.v_proj.bias": "model.safetensors",
|
| 242 |
+
"audio_tower.layers.24.self_attn.v_proj.weight": "model.safetensors",
|
| 243 |
+
"audio_tower.layers.24.self_attn_layer_norm.weight": "model.safetensors",
|
| 244 |
+
"audio_tower.layers.25.final_layer_norm.weight": "model.safetensors",
|
| 245 |
+
"audio_tower.layers.25.mlp.down_proj.bias": "model.safetensors",
|
| 246 |
+
"audio_tower.layers.25.mlp.down_proj.weight": "model.safetensors",
|
| 247 |
+
"audio_tower.layers.25.mlp.gate_proj.weight": "model.safetensors",
|
| 248 |
+
"audio_tower.layers.25.mlp.up_proj.weight": "model.safetensors",
|
| 249 |
+
"audio_tower.layers.25.self_attn.k_proj.weight": "model.safetensors",
|
| 250 |
+
"audio_tower.layers.25.self_attn.o_proj.bias": "model.safetensors",
|
| 251 |
+
"audio_tower.layers.25.self_attn.o_proj.weight": "model.safetensors",
|
| 252 |
+
"audio_tower.layers.25.self_attn.q_proj.bias": "model.safetensors",
|
| 253 |
+
"audio_tower.layers.25.self_attn.q_proj.weight": "model.safetensors",
|
| 254 |
+
"audio_tower.layers.25.self_attn.v_proj.bias": "model.safetensors",
|
| 255 |
+
"audio_tower.layers.25.self_attn.v_proj.weight": "model.safetensors",
|
| 256 |
+
"audio_tower.layers.25.self_attn_layer_norm.weight": "model.safetensors",
|
| 257 |
+
"audio_tower.layers.26.final_layer_norm.weight": "model.safetensors",
|
| 258 |
+
"audio_tower.layers.26.mlp.down_proj.bias": "model.safetensors",
|
| 259 |
+
"audio_tower.layers.26.mlp.down_proj.weight": "model.safetensors",
|
| 260 |
+
"audio_tower.layers.26.mlp.gate_proj.weight": "model.safetensors",
|
| 261 |
+
"audio_tower.layers.26.mlp.up_proj.weight": "model.safetensors",
|
| 262 |
+
"audio_tower.layers.26.self_attn.k_proj.weight": "model.safetensors",
|
| 263 |
+
"audio_tower.layers.26.self_attn.o_proj.bias": "model.safetensors",
|
| 264 |
+
"audio_tower.layers.26.self_attn.o_proj.weight": "model.safetensors",
|
| 265 |
+
"audio_tower.layers.26.self_attn.q_proj.bias": "model.safetensors",
|
| 266 |
+
"audio_tower.layers.26.self_attn.q_proj.weight": "model.safetensors",
|
| 267 |
+
"audio_tower.layers.26.self_attn.v_proj.bias": "model.safetensors",
|
| 268 |
+
"audio_tower.layers.26.self_attn.v_proj.weight": "model.safetensors",
|
| 269 |
+
"audio_tower.layers.26.self_attn_layer_norm.weight": "model.safetensors",
|
| 270 |
+
"audio_tower.layers.27.final_layer_norm.weight": "model.safetensors",
|
| 271 |
+
"audio_tower.layers.27.mlp.down_proj.bias": "model.safetensors",
|
| 272 |
+
"audio_tower.layers.27.mlp.down_proj.weight": "model.safetensors",
|
| 273 |
+
"audio_tower.layers.27.mlp.gate_proj.weight": "model.safetensors",
|
| 274 |
+
"audio_tower.layers.27.mlp.up_proj.weight": "model.safetensors",
|
| 275 |
+
"audio_tower.layers.27.self_attn.k_proj.weight": "model.safetensors",
|
| 276 |
+
"audio_tower.layers.27.self_attn.o_proj.bias": "model.safetensors",
|
| 277 |
+
"audio_tower.layers.27.self_attn.o_proj.weight": "model.safetensors",
|
| 278 |
+
"audio_tower.layers.27.self_attn.q_proj.bias": "model.safetensors",
|
| 279 |
+
"audio_tower.layers.27.self_attn.q_proj.weight": "model.safetensors",
|
| 280 |
+
"audio_tower.layers.27.self_attn.v_proj.bias": "model.safetensors",
|
| 281 |
+
"audio_tower.layers.27.self_attn.v_proj.weight": "model.safetensors",
|
| 282 |
+
"audio_tower.layers.27.self_attn_layer_norm.weight": "model.safetensors",
|
| 283 |
+
"audio_tower.layers.28.final_layer_norm.weight": "model.safetensors",
|
| 284 |
+
"audio_tower.layers.28.mlp.down_proj.bias": "model.safetensors",
|
| 285 |
+
"audio_tower.layers.28.mlp.down_proj.weight": "model.safetensors",
|
| 286 |
+
"audio_tower.layers.28.mlp.gate_proj.weight": "model.safetensors",
|
| 287 |
+
"audio_tower.layers.28.mlp.up_proj.weight": "model.safetensors",
|
| 288 |
+
"audio_tower.layers.28.self_attn.k_proj.weight": "model.safetensors",
|
| 289 |
+
"audio_tower.layers.28.self_attn.o_proj.bias": "model.safetensors",
|
| 290 |
+
"audio_tower.layers.28.self_attn.o_proj.weight": "model.safetensors",
|
| 291 |
+
"audio_tower.layers.28.self_attn.q_proj.bias": "model.safetensors",
|
| 292 |
+
"audio_tower.layers.28.self_attn.q_proj.weight": "model.safetensors",
|
| 293 |
+
"audio_tower.layers.28.self_attn.v_proj.bias": "model.safetensors",
|
| 294 |
+
"audio_tower.layers.28.self_attn.v_proj.weight": "model.safetensors",
|
| 295 |
+
"audio_tower.layers.28.self_attn_layer_norm.weight": "model.safetensors",
|
| 296 |
+
"audio_tower.layers.29.final_layer_norm.weight": "model.safetensors",
|
| 297 |
+
"audio_tower.layers.29.mlp.down_proj.bias": "model.safetensors",
|
| 298 |
+
"audio_tower.layers.29.mlp.down_proj.weight": "model.safetensors",
|
| 299 |
+
"audio_tower.layers.29.mlp.gate_proj.weight": "model.safetensors",
|
| 300 |
+
"audio_tower.layers.29.mlp.up_proj.weight": "model.safetensors",
|
| 301 |
+
"audio_tower.layers.29.self_attn.k_proj.weight": "model.safetensors",
|
| 302 |
+
"audio_tower.layers.29.self_attn.o_proj.bias": "model.safetensors",
|
| 303 |
+
"audio_tower.layers.29.self_attn.o_proj.weight": "model.safetensors",
|
| 304 |
+
"audio_tower.layers.29.self_attn.q_proj.bias": "model.safetensors",
|
| 305 |
+
"audio_tower.layers.29.self_attn.q_proj.weight": "model.safetensors",
|
| 306 |
+
"audio_tower.layers.29.self_attn.v_proj.bias": "model.safetensors",
|
| 307 |
+
"audio_tower.layers.29.self_attn.v_proj.weight": "model.safetensors",
|
| 308 |
+
"audio_tower.layers.29.self_attn_layer_norm.weight": "model.safetensors",
|
| 309 |
+
"audio_tower.layers.3.final_layer_norm.weight": "model.safetensors",
|
| 310 |
+
"audio_tower.layers.3.mlp.down_proj.bias": "model.safetensors",
|
| 311 |
+
"audio_tower.layers.3.mlp.down_proj.weight": "model.safetensors",
|
| 312 |
+
"audio_tower.layers.3.mlp.gate_proj.weight": "model.safetensors",
|
| 313 |
+
"audio_tower.layers.3.mlp.up_proj.weight": "model.safetensors",
|
| 314 |
+
"audio_tower.layers.3.self_attn.k_proj.weight": "model.safetensors",
|
| 315 |
+
"audio_tower.layers.3.self_attn.o_proj.bias": "model.safetensors",
|
| 316 |
+
"audio_tower.layers.3.self_attn.o_proj.weight": "model.safetensors",
|
| 317 |
+
"audio_tower.layers.3.self_attn.q_proj.bias": "model.safetensors",
|
| 318 |
+
"audio_tower.layers.3.self_attn.q_proj.weight": "model.safetensors",
|
| 319 |
+
"audio_tower.layers.3.self_attn.v_proj.bias": "model.safetensors",
|
| 320 |
+
"audio_tower.layers.3.self_attn.v_proj.weight": "model.safetensors",
|
| 321 |
+
"audio_tower.layers.3.self_attn_layer_norm.weight": "model.safetensors",
|
| 322 |
+
"audio_tower.layers.30.final_layer_norm.weight": "model.safetensors",
|
| 323 |
+
"audio_tower.layers.30.mlp.down_proj.bias": "model.safetensors",
|
| 324 |
+
"audio_tower.layers.30.mlp.down_proj.weight": "model.safetensors",
|
| 325 |
+
"audio_tower.layers.30.mlp.gate_proj.weight": "model.safetensors",
|
| 326 |
+
"audio_tower.layers.30.mlp.up_proj.weight": "model.safetensors",
|
| 327 |
+
"audio_tower.layers.30.self_attn.k_proj.weight": "model.safetensors",
|
| 328 |
+
"audio_tower.layers.30.self_attn.o_proj.bias": "model.safetensors",
|
| 329 |
+
"audio_tower.layers.30.self_attn.o_proj.weight": "model.safetensors",
|
| 330 |
+
"audio_tower.layers.30.self_attn.q_proj.bias": "model.safetensors",
|
| 331 |
+
"audio_tower.layers.30.self_attn.q_proj.weight": "model.safetensors",
|
| 332 |
+
"audio_tower.layers.30.self_attn.v_proj.bias": "model.safetensors",
|
| 333 |
+
"audio_tower.layers.30.self_attn.v_proj.weight": "model.safetensors",
|
| 334 |
+
"audio_tower.layers.30.self_attn_layer_norm.weight": "model.safetensors",
|
| 335 |
+
"audio_tower.layers.31.final_layer_norm.weight": "model.safetensors",
|
| 336 |
+
"audio_tower.layers.31.mlp.down_proj.bias": "model.safetensors",
|
| 337 |
+
"audio_tower.layers.31.mlp.down_proj.weight": "model.safetensors",
|
| 338 |
+
"audio_tower.layers.31.mlp.gate_proj.weight": "model.safetensors",
|
| 339 |
+
"audio_tower.layers.31.mlp.up_proj.weight": "model.safetensors",
|
| 340 |
+
"audio_tower.layers.31.self_attn.k_proj.weight": "model.safetensors",
|
| 341 |
+
"audio_tower.layers.31.self_attn.o_proj.bias": "model.safetensors",
|
| 342 |
+
"audio_tower.layers.31.self_attn.o_proj.weight": "model.safetensors",
|
| 343 |
+
"audio_tower.layers.31.self_attn.q_proj.bias": "model.safetensors",
|
| 344 |
+
"audio_tower.layers.31.self_attn.q_proj.weight": "model.safetensors",
|
| 345 |
+
"audio_tower.layers.31.self_attn.v_proj.bias": "model.safetensors",
|
| 346 |
+
"audio_tower.layers.31.self_attn.v_proj.weight": "model.safetensors",
|
| 347 |
+
"audio_tower.layers.31.self_attn_layer_norm.weight": "model.safetensors",
|
| 348 |
+
"audio_tower.layers.4.final_layer_norm.weight": "model.safetensors",
|
| 349 |
+
"audio_tower.layers.4.mlp.down_proj.bias": "model.safetensors",
|
| 350 |
+
"audio_tower.layers.4.mlp.down_proj.weight": "model.safetensors",
|
| 351 |
+
"audio_tower.layers.4.mlp.gate_proj.weight": "model.safetensors",
|
| 352 |
+
"audio_tower.layers.4.mlp.up_proj.weight": "model.safetensors",
|
| 353 |
+
"audio_tower.layers.4.self_attn.k_proj.weight": "model.safetensors",
|
| 354 |
+
"audio_tower.layers.4.self_attn.o_proj.bias": "model.safetensors",
|
| 355 |
+
"audio_tower.layers.4.self_attn.o_proj.weight": "model.safetensors",
|
| 356 |
+
"audio_tower.layers.4.self_attn.q_proj.bias": "model.safetensors",
|
| 357 |
+
"audio_tower.layers.4.self_attn.q_proj.weight": "model.safetensors",
|
| 358 |
+
"audio_tower.layers.4.self_attn.v_proj.bias": "model.safetensors",
|
| 359 |
+
"audio_tower.layers.4.self_attn.v_proj.weight": "model.safetensors",
|
| 360 |
+
"audio_tower.layers.4.self_attn_layer_norm.weight": "model.safetensors",
|
| 361 |
+
"audio_tower.layers.5.final_layer_norm.weight": "model.safetensors",
|
| 362 |
+
"audio_tower.layers.5.mlp.down_proj.bias": "model.safetensors",
|
| 363 |
+
"audio_tower.layers.5.mlp.down_proj.weight": "model.safetensors",
|
| 364 |
+
"audio_tower.layers.5.mlp.gate_proj.weight": "model.safetensors",
|
| 365 |
+
"audio_tower.layers.5.mlp.up_proj.weight": "model.safetensors",
|
| 366 |
+
"audio_tower.layers.5.self_attn.k_proj.weight": "model.safetensors",
|
| 367 |
+
"audio_tower.layers.5.self_attn.o_proj.bias": "model.safetensors",
|
| 368 |
+
"audio_tower.layers.5.self_attn.o_proj.weight": "model.safetensors",
|
| 369 |
+
"audio_tower.layers.5.self_attn.q_proj.bias": "model.safetensors",
|
| 370 |
+
"audio_tower.layers.5.self_attn.q_proj.weight": "model.safetensors",
|
| 371 |
+
"audio_tower.layers.5.self_attn.v_proj.bias": "model.safetensors",
|
| 372 |
+
"audio_tower.layers.5.self_attn.v_proj.weight": "model.safetensors",
|
| 373 |
+
"audio_tower.layers.5.self_attn_layer_norm.weight": "model.safetensors",
|
| 374 |
+
"audio_tower.layers.6.final_layer_norm.weight": "model.safetensors",
|
| 375 |
+
"audio_tower.layers.6.mlp.down_proj.bias": "model.safetensors",
|
| 376 |
+
"audio_tower.layers.6.mlp.down_proj.weight": "model.safetensors",
|
| 377 |
+
"audio_tower.layers.6.mlp.gate_proj.weight": "model.safetensors",
|
| 378 |
+
"audio_tower.layers.6.mlp.up_proj.weight": "model.safetensors",
|
| 379 |
+
"audio_tower.layers.6.self_attn.k_proj.weight": "model.safetensors",
|
| 380 |
+
"audio_tower.layers.6.self_attn.o_proj.bias": "model.safetensors",
|
| 381 |
+
"audio_tower.layers.6.self_attn.o_proj.weight": "model.safetensors",
|
| 382 |
+
"audio_tower.layers.6.self_attn.q_proj.bias": "model.safetensors",
|
| 383 |
+
"audio_tower.layers.6.self_attn.q_proj.weight": "model.safetensors",
|
| 384 |
+
"audio_tower.layers.6.self_attn.v_proj.bias": "model.safetensors",
|
| 385 |
+
"audio_tower.layers.6.self_attn.v_proj.weight": "model.safetensors",
|
| 386 |
+
"audio_tower.layers.6.self_attn_layer_norm.weight": "model.safetensors",
|
| 387 |
+
"audio_tower.layers.7.final_layer_norm.weight": "model.safetensors",
|
| 388 |
+
"audio_tower.layers.7.mlp.down_proj.bias": "model.safetensors",
|
| 389 |
+
"audio_tower.layers.7.mlp.down_proj.weight": "model.safetensors",
|
| 390 |
+
"audio_tower.layers.7.mlp.gate_proj.weight": "model.safetensors",
|
| 391 |
+
"audio_tower.layers.7.mlp.up_proj.weight": "model.safetensors",
|
| 392 |
+
"audio_tower.layers.7.self_attn.k_proj.weight": "model.safetensors",
|
| 393 |
+
"audio_tower.layers.7.self_attn.o_proj.bias": "model.safetensors",
|
| 394 |
+
"audio_tower.layers.7.self_attn.o_proj.weight": "model.safetensors",
|
| 395 |
+
"audio_tower.layers.7.self_attn.q_proj.bias": "model.safetensors",
|
| 396 |
+
"audio_tower.layers.7.self_attn.q_proj.weight": "model.safetensors",
|
| 397 |
+
"audio_tower.layers.7.self_attn.v_proj.bias": "model.safetensors",
|
| 398 |
+
"audio_tower.layers.7.self_attn.v_proj.weight": "model.safetensors",
|
| 399 |
+
"audio_tower.layers.7.self_attn_layer_norm.weight": "model.safetensors",
|
| 400 |
+
"audio_tower.layers.8.final_layer_norm.weight": "model.safetensors",
|
| 401 |
+
"audio_tower.layers.8.mlp.down_proj.bias": "model.safetensors",
|
| 402 |
+
"audio_tower.layers.8.mlp.down_proj.weight": "model.safetensors",
|
| 403 |
+
"audio_tower.layers.8.mlp.gate_proj.weight": "model.safetensors",
|
| 404 |
+
"audio_tower.layers.8.mlp.up_proj.weight": "model.safetensors",
|
| 405 |
+
"audio_tower.layers.8.self_attn.k_proj.weight": "model.safetensors",
|
| 406 |
+
"audio_tower.layers.8.self_attn.o_proj.bias": "model.safetensors",
|
| 407 |
+
"audio_tower.layers.8.self_attn.o_proj.weight": "model.safetensors",
|
| 408 |
+
"audio_tower.layers.8.self_attn.q_proj.bias": "model.safetensors",
|
| 409 |
+
"audio_tower.layers.8.self_attn.q_proj.weight": "model.safetensors",
|
| 410 |
+
"audio_tower.layers.8.self_attn.v_proj.bias": "model.safetensors",
|
| 411 |
+
"audio_tower.layers.8.self_attn.v_proj.weight": "model.safetensors",
|
| 412 |
+
"audio_tower.layers.8.self_attn_layer_norm.weight": "model.safetensors",
|
| 413 |
+
"audio_tower.layers.9.final_layer_norm.weight": "model.safetensors",
|
| 414 |
+
"audio_tower.layers.9.mlp.down_proj.bias": "model.safetensors",
|
| 415 |
+
"audio_tower.layers.9.mlp.down_proj.weight": "model.safetensors",
|
| 416 |
+
"audio_tower.layers.9.mlp.gate_proj.weight": "model.safetensors",
|
| 417 |
+
"audio_tower.layers.9.mlp.up_proj.weight": "model.safetensors",
|
| 418 |
+
"audio_tower.layers.9.self_attn.k_proj.weight": "model.safetensors",
|
| 419 |
+
"audio_tower.layers.9.self_attn.o_proj.bias": "model.safetensors",
|
| 420 |
+
"audio_tower.layers.9.self_attn.o_proj.weight": "model.safetensors",
|
| 421 |
+
"audio_tower.layers.9.self_attn.q_proj.bias": "model.safetensors",
|
| 422 |
+
"audio_tower.layers.9.self_attn.q_proj.weight": "model.safetensors",
|
| 423 |
+
"audio_tower.layers.9.self_attn.v_proj.bias": "model.safetensors",
|
| 424 |
+
"audio_tower.layers.9.self_attn.v_proj.weight": "model.safetensors",
|
| 425 |
+
"audio_tower.layers.9.self_attn_layer_norm.weight": "model.safetensors",
|
| 426 |
+
"audio_tower.norm.weight": "model.safetensors",
|
| 427 |
+
"frame_len_embedding.weight": "model.safetensors",
|
| 428 |
+
"language_model.model.embed_tokens.weight": "model.safetensors",
|
| 429 |
+
"language_model.model.layers.0.ada_rms_norm.linear1.weight": "model.safetensors",
|
| 430 |
+
"language_model.model.layers.0.ada_rms_norm.linear2.weight": "model.safetensors",
|
| 431 |
+
"language_model.model.layers.0.input_layernorm.weight": "model.safetensors",
|
| 432 |
+
"language_model.model.layers.0.mlp.down_proj.weight": "model.safetensors",
|
| 433 |
+
"language_model.model.layers.0.mlp.gate_proj.weight": "model.safetensors",
|
| 434 |
+
"language_model.model.layers.0.mlp.up_proj.weight": "model.safetensors",
|
| 435 |
+
"language_model.model.layers.0.post_attention_layernorm.weight": "model.safetensors",
|
| 436 |
+
"language_model.model.layers.0.self_attn.k_proj.bias": "model.safetensors",
|
| 437 |
+
"language_model.model.layers.0.self_attn.k_proj.weight": "model.safetensors",
|
| 438 |
+
"language_model.model.layers.0.self_attn.o_proj.weight": "model.safetensors",
|
| 439 |
+
"language_model.model.layers.0.self_attn.q_proj.bias": "model.safetensors",
|
| 440 |
+
"language_model.model.layers.0.self_attn.q_proj.weight": "model.safetensors",
|
| 441 |
+
"language_model.model.layers.0.self_attn.v_proj.bias": "model.safetensors",
|
| 442 |
+
"language_model.model.layers.0.self_attn.v_proj.weight": "model.safetensors",
|
| 443 |
+
"language_model.model.layers.1.ada_rms_norm.linear1.weight": "model.safetensors",
|
| 444 |
+
"language_model.model.layers.1.ada_rms_norm.linear2.weight": "model.safetensors",
|
| 445 |
+
"language_model.model.layers.1.input_layernorm.weight": "model.safetensors",
|
| 446 |
+
"language_model.model.layers.1.mlp.down_proj.weight": "model.safetensors",
|
| 447 |
+
"language_model.model.layers.1.mlp.gate_proj.weight": "model.safetensors",
|
| 448 |
+
"language_model.model.layers.1.mlp.up_proj.weight": "model.safetensors",
|
| 449 |
+
"language_model.model.layers.1.post_attention_layernorm.weight": "model.safetensors",
|
| 450 |
+
"language_model.model.layers.1.self_attn.k_proj.bias": "model.safetensors",
|
| 451 |
+
"language_model.model.layers.1.self_attn.k_proj.weight": "model.safetensors",
|
| 452 |
+
"language_model.model.layers.1.self_attn.o_proj.weight": "model.safetensors",
|
| 453 |
+
"language_model.model.layers.1.self_attn.q_proj.bias": "model.safetensors",
|
| 454 |
+
"language_model.model.layers.1.self_attn.q_proj.weight": "model.safetensors",
|
| 455 |
+
"language_model.model.layers.1.self_attn.v_proj.bias": "model.safetensors",
|
| 456 |
+
"language_model.model.layers.1.self_attn.v_proj.weight": "model.safetensors",
|
| 457 |
+
"language_model.model.layers.10.ada_rms_norm.linear1.weight": "model.safetensors",
|
| 458 |
+
"language_model.model.layers.10.ada_rms_norm.linear2.weight": "model.safetensors",
|
| 459 |
+
"language_model.model.layers.10.input_layernorm.weight": "model.safetensors",
|
| 460 |
+
"language_model.model.layers.10.mlp.down_proj.weight": "model.safetensors",
|
| 461 |
+
"language_model.model.layers.10.mlp.gate_proj.weight": "model.safetensors",
|
| 462 |
+
"language_model.model.layers.10.mlp.up_proj.weight": "model.safetensors",
|
| 463 |
+
"language_model.model.layers.10.post_attention_layernorm.weight": "model.safetensors",
|
| 464 |
+
"language_model.model.layers.10.self_attn.k_proj.bias": "model.safetensors",
|
| 465 |
+
"language_model.model.layers.10.self_attn.k_proj.weight": "model.safetensors",
|
| 466 |
+
"language_model.model.layers.10.self_attn.o_proj.weight": "model.safetensors",
|
| 467 |
+
"language_model.model.layers.10.self_attn.q_proj.bias": "model.safetensors",
|
| 468 |
+
"language_model.model.layers.10.self_attn.q_proj.weight": "model.safetensors",
|
| 469 |
+
"language_model.model.layers.10.self_attn.v_proj.bias": "model.safetensors",
|
| 470 |
+
"language_model.model.layers.10.self_attn.v_proj.weight": "model.safetensors",
|
| 471 |
+
"language_model.model.layers.11.ada_rms_norm.linear1.weight": "model.safetensors",
|
| 472 |
+
"language_model.model.layers.11.ada_rms_norm.linear2.weight": "model.safetensors",
|
| 473 |
+
"language_model.model.layers.11.input_layernorm.weight": "model.safetensors",
|
| 474 |
+
"language_model.model.layers.11.mlp.down_proj.weight": "model.safetensors",
|
| 475 |
+
"language_model.model.layers.11.mlp.gate_proj.weight": "model.safetensors",
|
| 476 |
+
"language_model.model.layers.11.mlp.up_proj.weight": "model.safetensors",
|
| 477 |
+
"language_model.model.layers.11.post_attention_layernorm.weight": "model.safetensors",
|
| 478 |
+
"language_model.model.layers.11.self_attn.k_proj.bias": "model.safetensors",
|
| 479 |
+
"language_model.model.layers.11.self_attn.k_proj.weight": "model.safetensors",
|
| 480 |
+
"language_model.model.layers.11.self_attn.o_proj.weight": "model.safetensors",
|
| 481 |
+
"language_model.model.layers.11.self_attn.q_proj.bias": "model.safetensors",
|
| 482 |
+
"language_model.model.layers.11.self_attn.q_proj.weight": "model.safetensors",
|
| 483 |
+
"language_model.model.layers.11.self_attn.v_proj.bias": "model.safetensors",
|
| 484 |
+
"language_model.model.layers.11.self_attn.v_proj.weight": "model.safetensors",
|
| 485 |
+
"language_model.model.layers.12.ada_rms_norm.linear1.weight": "model.safetensors",
|
| 486 |
+
"language_model.model.layers.12.ada_rms_norm.linear2.weight": "model.safetensors",
|
| 487 |
+
"language_model.model.layers.12.input_layernorm.weight": "model.safetensors",
|
| 488 |
+
"language_model.model.layers.12.mlp.down_proj.weight": "model.safetensors",
|
| 489 |
+
"language_model.model.layers.12.mlp.gate_proj.weight": "model.safetensors",
|
| 490 |
+
"language_model.model.layers.12.mlp.up_proj.weight": "model.safetensors",
|
| 491 |
+
"language_model.model.layers.12.post_attention_layernorm.weight": "model.safetensors",
|
| 492 |
+
"language_model.model.layers.12.self_attn.k_proj.bias": "model.safetensors",
|
| 493 |
+
"language_model.model.layers.12.self_attn.k_proj.weight": "model.safetensors",
|
| 494 |
+
"language_model.model.layers.12.self_attn.o_proj.weight": "model.safetensors",
|
| 495 |
+
"language_model.model.layers.12.self_attn.q_proj.bias": "model.safetensors",
|
| 496 |
+
"language_model.model.layers.12.self_attn.q_proj.weight": "model.safetensors",
|
| 497 |
+
"language_model.model.layers.12.self_attn.v_proj.bias": "model.safetensors",
|
| 498 |
+
"language_model.model.layers.12.self_attn.v_proj.weight": "model.safetensors",
|
| 499 |
+
"language_model.model.layers.13.ada_rms_norm.linear1.weight": "model.safetensors",
|
| 500 |
+
"language_model.model.layers.13.ada_rms_norm.linear2.weight": "model.safetensors",
|
| 501 |
+
"language_model.model.layers.13.input_layernorm.weight": "model.safetensors",
|
| 502 |
+
"language_model.model.layers.13.mlp.down_proj.weight": "model.safetensors",
|
| 503 |
+
"language_model.model.layers.13.mlp.gate_proj.weight": "model.safetensors",
|
| 504 |
+
"language_model.model.layers.13.mlp.up_proj.weight": "model.safetensors",
|
| 505 |
+
"language_model.model.layers.13.post_attention_layernorm.weight": "model.safetensors",
|
| 506 |
+
"language_model.model.layers.13.self_attn.k_proj.bias": "model.safetensors",
|
| 507 |
+
"language_model.model.layers.13.self_attn.k_proj.weight": "model.safetensors",
|
| 508 |
+
"language_model.model.layers.13.self_attn.o_proj.weight": "model.safetensors",
|
| 509 |
+
"language_model.model.layers.13.self_attn.q_proj.bias": "model.safetensors",
|
| 510 |
+
"language_model.model.layers.13.self_attn.q_proj.weight": "model.safetensors",
|
| 511 |
+
"language_model.model.layers.13.self_attn.v_proj.bias": "model.safetensors",
|
| 512 |
+
"language_model.model.layers.13.self_attn.v_proj.weight": "model.safetensors",
|
| 513 |
+
"language_model.model.layers.14.ada_rms_norm.linear1.weight": "model.safetensors",
|
| 514 |
+
"language_model.model.layers.14.ada_rms_norm.linear2.weight": "model.safetensors",
|
| 515 |
+
"language_model.model.layers.14.input_layernorm.weight": "model.safetensors",
|
| 516 |
+
"language_model.model.layers.14.mlp.down_proj.weight": "model.safetensors",
|
| 517 |
+
"language_model.model.layers.14.mlp.gate_proj.weight": "model.safetensors",
|
| 518 |
+
"language_model.model.layers.14.mlp.up_proj.weight": "model.safetensors",
|
| 519 |
+
"language_model.model.layers.14.post_attention_layernorm.weight": "model.safetensors",
|
| 520 |
+
"language_model.model.layers.14.self_attn.k_proj.bias": "model.safetensors",
|
| 521 |
+
"language_model.model.layers.14.self_attn.k_proj.weight": "model.safetensors",
|
| 522 |
+
"language_model.model.layers.14.self_attn.o_proj.weight": "model.safetensors",
|
| 523 |
+
"language_model.model.layers.14.self_attn.q_proj.bias": "model.safetensors",
|
| 524 |
+
"language_model.model.layers.14.self_attn.q_proj.weight": "model.safetensors",
|
| 525 |
+
"language_model.model.layers.14.self_attn.v_proj.bias": "model.safetensors",
|
| 526 |
+
"language_model.model.layers.14.self_attn.v_proj.weight": "model.safetensors",
|
| 527 |
+
"language_model.model.layers.15.ada_rms_norm.linear1.weight": "model.safetensors",
|
| 528 |
+
"language_model.model.layers.15.ada_rms_norm.linear2.weight": "model.safetensors",
|
| 529 |
+
"language_model.model.layers.15.input_layernorm.weight": "model.safetensors",
|
| 530 |
+
"language_model.model.layers.15.mlp.down_proj.weight": "model.safetensors",
|
| 531 |
+
"language_model.model.layers.15.mlp.gate_proj.weight": "model.safetensors",
|
| 532 |
+
"language_model.model.layers.15.mlp.up_proj.weight": "model.safetensors",
|
| 533 |
+
"language_model.model.layers.15.post_attention_layernorm.weight": "model.safetensors",
|
| 534 |
+
"language_model.model.layers.15.self_attn.k_proj.bias": "model.safetensors",
|
| 535 |
+
"language_model.model.layers.15.self_attn.k_proj.weight": "model.safetensors",
|
| 536 |
+
"language_model.model.layers.15.self_attn.o_proj.weight": "model.safetensors",
|
| 537 |
+
"language_model.model.layers.15.self_attn.q_proj.bias": "model.safetensors",
|
| 538 |
+
"language_model.model.layers.15.self_attn.q_proj.weight": "model.safetensors",
|
| 539 |
+
"language_model.model.layers.15.self_attn.v_proj.bias": "model.safetensors",
|
| 540 |
+
"language_model.model.layers.15.self_attn.v_proj.weight": "model.safetensors",
|
| 541 |
+
"language_model.model.layers.16.ada_rms_norm.linear1.weight": "model.safetensors",
|
| 542 |
+
"language_model.model.layers.16.ada_rms_norm.linear2.weight": "model.safetensors",
|
| 543 |
+
"language_model.model.layers.16.input_layernorm.weight": "model.safetensors",
|
| 544 |
+
"language_model.model.layers.16.mlp.down_proj.weight": "model.safetensors",
|
| 545 |
+
"language_model.model.layers.16.mlp.gate_proj.weight": "model.safetensors",
|
| 546 |
+
"language_model.model.layers.16.mlp.up_proj.weight": "model.safetensors",
|
| 547 |
+
"language_model.model.layers.16.post_attention_layernorm.weight": "model.safetensors",
|
| 548 |
+
"language_model.model.layers.16.self_attn.k_proj.bias": "model.safetensors",
|
| 549 |
+
"language_model.model.layers.16.self_attn.k_proj.weight": "model.safetensors",
|
| 550 |
+
"language_model.model.layers.16.self_attn.o_proj.weight": "model.safetensors",
|
| 551 |
+
"language_model.model.layers.16.self_attn.q_proj.bias": "model.safetensors",
|
| 552 |
+
"language_model.model.layers.16.self_attn.q_proj.weight": "model.safetensors",
|
| 553 |
+
"language_model.model.layers.16.self_attn.v_proj.bias": "model.safetensors",
|
| 554 |
+
"language_model.model.layers.16.self_attn.v_proj.weight": "model.safetensors",
|
| 555 |
+
"language_model.model.layers.17.ada_rms_norm.linear1.weight": "model.safetensors",
|
| 556 |
+
"language_model.model.layers.17.ada_rms_norm.linear2.weight": "model.safetensors",
|
| 557 |
+
"language_model.model.layers.17.input_layernorm.weight": "model.safetensors",
|
| 558 |
+
"language_model.model.layers.17.mlp.down_proj.weight": "model.safetensors",
|
| 559 |
+
"language_model.model.layers.17.mlp.gate_proj.weight": "model.safetensors",
|
| 560 |
+
"language_model.model.layers.17.mlp.up_proj.weight": "model.safetensors",
|
| 561 |
+
"language_model.model.layers.17.post_attention_layernorm.weight": "model.safetensors",
|
| 562 |
+
"language_model.model.layers.17.self_attn.k_proj.bias": "model.safetensors",
|
| 563 |
+
"language_model.model.layers.17.self_attn.k_proj.weight": "model.safetensors",
|
| 564 |
+
"language_model.model.layers.17.self_attn.o_proj.weight": "model.safetensors",
|
| 565 |
+
"language_model.model.layers.17.self_attn.q_proj.bias": "model.safetensors",
|
| 566 |
+
"language_model.model.layers.17.self_attn.q_proj.weight": "model.safetensors",
|
| 567 |
+
"language_model.model.layers.17.self_attn.v_proj.bias": "model.safetensors",
|
| 568 |
+
"language_model.model.layers.17.self_attn.v_proj.weight": "model.safetensors",
|
| 569 |
+
"language_model.model.layers.18.ada_rms_norm.linear1.weight": "model.safetensors",
|
| 570 |
+
"language_model.model.layers.18.ada_rms_norm.linear2.weight": "model.safetensors",
|
| 571 |
+
"language_model.model.layers.18.input_layernorm.weight": "model.safetensors",
|
| 572 |
+
"language_model.model.layers.18.mlp.down_proj.weight": "model.safetensors",
|
| 573 |
+
"language_model.model.layers.18.mlp.gate_proj.weight": "model.safetensors",
|
| 574 |
+
"language_model.model.layers.18.mlp.up_proj.weight": "model.safetensors",
|
| 575 |
+
"language_model.model.layers.18.post_attention_layernorm.weight": "model.safetensors",
|
| 576 |
+
"language_model.model.layers.18.self_attn.k_proj.bias": "model.safetensors",
|
| 577 |
+
"language_model.model.layers.18.self_attn.k_proj.weight": "model.safetensors",
|
| 578 |
+
"language_model.model.layers.18.self_attn.o_proj.weight": "model.safetensors",
|
| 579 |
+
"language_model.model.layers.18.self_attn.q_proj.bias": "model.safetensors",
|
| 580 |
+
"language_model.model.layers.18.self_attn.q_proj.weight": "model.safetensors",
|
| 581 |
+
"language_model.model.layers.18.self_attn.v_proj.bias": "model.safetensors",
|
| 582 |
+
"language_model.model.layers.18.self_attn.v_proj.weight": "model.safetensors",
|
| 583 |
+
"language_model.model.layers.19.ada_rms_norm.linear1.weight": "model.safetensors",
|
| 584 |
+
"language_model.model.layers.19.ada_rms_norm.linear2.weight": "model.safetensors",
|
| 585 |
+
"language_model.model.layers.19.input_layernorm.weight": "model.safetensors",
|
| 586 |
+
"language_model.model.layers.19.mlp.down_proj.weight": "model.safetensors",
|
| 587 |
+
"language_model.model.layers.19.mlp.gate_proj.weight": "model.safetensors",
|
| 588 |
+
"language_model.model.layers.19.mlp.up_proj.weight": "model.safetensors",
|
| 589 |
+
"language_model.model.layers.19.post_attention_layernorm.weight": "model.safetensors",
|
| 590 |
+
"language_model.model.layers.19.self_attn.k_proj.bias": "model.safetensors",
|
| 591 |
+
"language_model.model.layers.19.self_attn.k_proj.weight": "model.safetensors",
|
| 592 |
+
"language_model.model.layers.19.self_attn.o_proj.weight": "model.safetensors",
|
| 593 |
+
"language_model.model.layers.19.self_attn.q_proj.bias": "model.safetensors",
|
| 594 |
+
"language_model.model.layers.19.self_attn.q_proj.weight": "model.safetensors",
|
| 595 |
+
"language_model.model.layers.19.self_attn.v_proj.bias": "model.safetensors",
|
| 596 |
+
"language_model.model.layers.19.self_attn.v_proj.weight": "model.safetensors",
|
| 597 |
+
"language_model.model.layers.2.ada_rms_norm.linear1.weight": "model.safetensors",
|
| 598 |
+
"language_model.model.layers.2.ada_rms_norm.linear2.weight": "model.safetensors",
|
| 599 |
+
"language_model.model.layers.2.input_layernorm.weight": "model.safetensors",
|
| 600 |
+
"language_model.model.layers.2.mlp.down_proj.weight": "model.safetensors",
|
| 601 |
+
"language_model.model.layers.2.mlp.gate_proj.weight": "model.safetensors",
|
| 602 |
+
"language_model.model.layers.2.mlp.up_proj.weight": "model.safetensors",
|
| 603 |
+
"language_model.model.layers.2.post_attention_layernorm.weight": "model.safetensors",
|
| 604 |
+
"language_model.model.layers.2.self_attn.k_proj.bias": "model.safetensors",
|
| 605 |
+
"language_model.model.layers.2.self_attn.k_proj.weight": "model.safetensors",
|
| 606 |
+
"language_model.model.layers.2.self_attn.o_proj.weight": "model.safetensors",
|
| 607 |
+
"language_model.model.layers.2.self_attn.q_proj.bias": "model.safetensors",
|
| 608 |
+
"language_model.model.layers.2.self_attn.q_proj.weight": "model.safetensors",
|
| 609 |
+
"language_model.model.layers.2.self_attn.v_proj.bias": "model.safetensors",
|
| 610 |
+
"language_model.model.layers.2.self_attn.v_proj.weight": "model.safetensors",
|
| 611 |
+
"language_model.model.layers.20.ada_rms_norm.linear1.weight": "model.safetensors",
|
| 612 |
+
"language_model.model.layers.20.ada_rms_norm.linear2.weight": "model.safetensors",
|
| 613 |
+
"language_model.model.layers.20.input_layernorm.weight": "model.safetensors",
|
| 614 |
+
"language_model.model.layers.20.mlp.down_proj.weight": "model.safetensors",
|
| 615 |
+
"language_model.model.layers.20.mlp.gate_proj.weight": "model.safetensors",
|
| 616 |
+
"language_model.model.layers.20.mlp.up_proj.weight": "model.safetensors",
|
| 617 |
+
"language_model.model.layers.20.post_attention_layernorm.weight": "model.safetensors",
|
| 618 |
+
"language_model.model.layers.20.self_attn.k_proj.bias": "model.safetensors",
|
| 619 |
+
"language_model.model.layers.20.self_attn.k_proj.weight": "model.safetensors",
|
| 620 |
+
"language_model.model.layers.20.self_attn.o_proj.weight": "model.safetensors",
|
| 621 |
+
"language_model.model.layers.20.self_attn.q_proj.bias": "model.safetensors",
|
| 622 |
+
"language_model.model.layers.20.self_attn.q_proj.weight": "model.safetensors",
|
| 623 |
+
"language_model.model.layers.20.self_attn.v_proj.bias": "model.safetensors",
|
| 624 |
+
"language_model.model.layers.20.self_attn.v_proj.weight": "model.safetensors",
|
| 625 |
+
"language_model.model.layers.21.ada_rms_norm.linear1.weight": "model.safetensors",
|
| 626 |
+
"language_model.model.layers.21.ada_rms_norm.linear2.weight": "model.safetensors",
|
| 627 |
+
"language_model.model.layers.21.input_layernorm.weight": "model.safetensors",
|
| 628 |
+
"language_model.model.layers.21.mlp.down_proj.weight": "model.safetensors",
|
| 629 |
+
"language_model.model.layers.21.mlp.gate_proj.weight": "model.safetensors",
|
| 630 |
+
"language_model.model.layers.21.mlp.up_proj.weight": "model.safetensors",
|
| 631 |
+
"language_model.model.layers.21.post_attention_layernorm.weight": "model.safetensors",
|
| 632 |
+
"language_model.model.layers.21.self_attn.k_proj.bias": "model.safetensors",
|
| 633 |
+
"language_model.model.layers.21.self_attn.k_proj.weight": "model.safetensors",
|
| 634 |
+
"language_model.model.layers.21.self_attn.o_proj.weight": "model.safetensors",
|
| 635 |
+
"language_model.model.layers.21.self_attn.q_proj.bias": "model.safetensors",
|
| 636 |
+
"language_model.model.layers.21.self_attn.q_proj.weight": "model.safetensors",
|
| 637 |
+
"language_model.model.layers.21.self_attn.v_proj.bias": "model.safetensors",
|
| 638 |
+
"language_model.model.layers.21.self_attn.v_proj.weight": "model.safetensors",
|
| 639 |
+
"language_model.model.layers.22.ada_rms_norm.linear1.weight": "model.safetensors",
|
| 640 |
+
"language_model.model.layers.22.ada_rms_norm.linear2.weight": "model.safetensors",
|
| 641 |
+
"language_model.model.layers.22.input_layernorm.weight": "model.safetensors",
|
| 642 |
+
"language_model.model.layers.22.mlp.down_proj.weight": "model.safetensors",
|
| 643 |
+
"language_model.model.layers.22.mlp.gate_proj.weight": "model.safetensors",
|
| 644 |
+
"language_model.model.layers.22.mlp.up_proj.weight": "model.safetensors",
|
| 645 |
+
"language_model.model.layers.22.post_attention_layernorm.weight": "model.safetensors",
|
| 646 |
+
"language_model.model.layers.22.self_attn.k_proj.bias": "model.safetensors",
|
| 647 |
+
"language_model.model.layers.22.self_attn.k_proj.weight": "model.safetensors",
|
| 648 |
+
"language_model.model.layers.22.self_attn.o_proj.weight": "model.safetensors",
|
| 649 |
+
"language_model.model.layers.22.self_attn.q_proj.bias": "model.safetensors",
|
| 650 |
+
"language_model.model.layers.22.self_attn.q_proj.weight": "model.safetensors",
|
| 651 |
+
"language_model.model.layers.22.self_attn.v_proj.bias": "model.safetensors",
|
| 652 |
+
"language_model.model.layers.22.self_attn.v_proj.weight": "model.safetensors",
|
| 653 |
+
"language_model.model.layers.23.ada_rms_norm.linear1.weight": "model.safetensors",
|
| 654 |
+
"language_model.model.layers.23.ada_rms_norm.linear2.weight": "model.safetensors",
|
| 655 |
+
"language_model.model.layers.23.input_layernorm.weight": "model.safetensors",
|
| 656 |
+
"language_model.model.layers.23.mlp.down_proj.weight": "model.safetensors",
|
| 657 |
+
"language_model.model.layers.23.mlp.gate_proj.weight": "model.safetensors",
|
| 658 |
+
"language_model.model.layers.23.mlp.up_proj.weight": "model.safetensors",
|
| 659 |
+
"language_model.model.layers.23.post_attention_layernorm.weight": "model.safetensors",
|
| 660 |
+
"language_model.model.layers.23.self_attn.k_proj.bias": "model.safetensors",
|
| 661 |
+
"language_model.model.layers.23.self_attn.k_proj.weight": "model.safetensors",
|
| 662 |
+
"language_model.model.layers.23.self_attn.o_proj.weight": "model.safetensors",
|
| 663 |
+
"language_model.model.layers.23.self_attn.q_proj.bias": "model.safetensors",
|
| 664 |
+
"language_model.model.layers.23.self_attn.q_proj.weight": "model.safetensors",
|
| 665 |
+
"language_model.model.layers.23.self_attn.v_proj.bias": "model.safetensors",
|
| 666 |
+
"language_model.model.layers.23.self_attn.v_proj.weight": "model.safetensors",
|
| 667 |
+
"language_model.model.layers.24.ada_rms_norm.linear1.weight": "model.safetensors",
|
| 668 |
+
"language_model.model.layers.24.ada_rms_norm.linear2.weight": "model.safetensors",
|
| 669 |
+
"language_model.model.layers.24.input_layernorm.weight": "model.safetensors",
|
| 670 |
+
"language_model.model.layers.24.mlp.down_proj.weight": "model.safetensors",
|
| 671 |
+
"language_model.model.layers.24.mlp.gate_proj.weight": "model.safetensors",
|
| 672 |
+
"language_model.model.layers.24.mlp.up_proj.weight": "model.safetensors",
|
| 673 |
+
"language_model.model.layers.24.post_attention_layernorm.weight": "model.safetensors",
|
| 674 |
+
"language_model.model.layers.24.self_attn.k_proj.bias": "model.safetensors",
|
| 675 |
+
"language_model.model.layers.24.self_attn.k_proj.weight": "model.safetensors",
|
| 676 |
+
"language_model.model.layers.24.self_attn.o_proj.weight": "model.safetensors",
|
| 677 |
+
"language_model.model.layers.24.self_attn.q_proj.bias": "model.safetensors",
|
| 678 |
+
"language_model.model.layers.24.self_attn.q_proj.weight": "model.safetensors",
|
| 679 |
+
"language_model.model.layers.24.self_attn.v_proj.bias": "model.safetensors",
|
| 680 |
+
"language_model.model.layers.24.self_attn.v_proj.weight": "model.safetensors",
|
| 681 |
+
"language_model.model.layers.25.ada_rms_norm.linear1.weight": "model.safetensors",
|
| 682 |
+
"language_model.model.layers.25.ada_rms_norm.linear2.weight": "model.safetensors",
|
| 683 |
+
"language_model.model.layers.25.input_layernorm.weight": "model.safetensors",
|
| 684 |
+
"language_model.model.layers.25.mlp.down_proj.weight": "model.safetensors",
|
| 685 |
+
"language_model.model.layers.25.mlp.gate_proj.weight": "model.safetensors",
|
| 686 |
+
"language_model.model.layers.25.mlp.up_proj.weight": "model.safetensors",
|
| 687 |
+
"language_model.model.layers.25.post_attention_layernorm.weight": "model.safetensors",
|
| 688 |
+
"language_model.model.layers.25.self_attn.k_proj.bias": "model.safetensors",
|
| 689 |
+
"language_model.model.layers.25.self_attn.k_proj.weight": "model.safetensors",
|
| 690 |
+
"language_model.model.layers.25.self_attn.o_proj.weight": "model.safetensors",
|
| 691 |
+
"language_model.model.layers.25.self_attn.q_proj.bias": "model.safetensors",
|
| 692 |
+
"language_model.model.layers.25.self_attn.q_proj.weight": "model.safetensors",
|
| 693 |
+
"language_model.model.layers.25.self_attn.v_proj.bias": "model.safetensors",
|
| 694 |
+
"language_model.model.layers.25.self_attn.v_proj.weight": "model.safetensors",
|
| 695 |
+
"language_model.model.layers.26.ada_rms_norm.linear1.weight": "model.safetensors",
|
| 696 |
+
"language_model.model.layers.26.ada_rms_norm.linear2.weight": "model.safetensors",
|
| 697 |
+
"language_model.model.layers.26.input_layernorm.weight": "model.safetensors",
|
| 698 |
+
"language_model.model.layers.26.mlp.down_proj.weight": "model.safetensors",
|
| 699 |
+
"language_model.model.layers.26.mlp.gate_proj.weight": "model.safetensors",
|
| 700 |
+
"language_model.model.layers.26.mlp.up_proj.weight": "model.safetensors",
|
| 701 |
+
"language_model.model.layers.26.post_attention_layernorm.weight": "model.safetensors",
|
| 702 |
+
"language_model.model.layers.26.self_attn.k_proj.bias": "model.safetensors",
|
| 703 |
+
"language_model.model.layers.26.self_attn.k_proj.weight": "model.safetensors",
|
| 704 |
+
"language_model.model.layers.26.self_attn.o_proj.weight": "model.safetensors",
|
| 705 |
+
"language_model.model.layers.26.self_attn.q_proj.bias": "model.safetensors",
|
| 706 |
+
"language_model.model.layers.26.self_attn.q_proj.weight": "model.safetensors",
|
| 707 |
+
"language_model.model.layers.26.self_attn.v_proj.bias": "model.safetensors",
|
| 708 |
+
"language_model.model.layers.26.self_attn.v_proj.weight": "model.safetensors",
|
| 709 |
+
"language_model.model.layers.27.ada_rms_norm.linear1.weight": "model.safetensors",
|
| 710 |
+
"language_model.model.layers.27.ada_rms_norm.linear2.weight": "model.safetensors",
|
| 711 |
+
"language_model.model.layers.27.input_layernorm.weight": "model.safetensors",
|
| 712 |
+
"language_model.model.layers.27.mlp.down_proj.weight": "model.safetensors",
|
| 713 |
+
"language_model.model.layers.27.mlp.gate_proj.weight": "model.safetensors",
|
| 714 |
+
"language_model.model.layers.27.mlp.up_proj.weight": "model.safetensors",
|
| 715 |
+
"language_model.model.layers.27.post_attention_layernorm.weight": "model.safetensors",
|
| 716 |
+
"language_model.model.layers.27.self_attn.k_proj.bias": "model.safetensors",
|
| 717 |
+
"language_model.model.layers.27.self_attn.k_proj.weight": "model.safetensors",
|
| 718 |
+
"language_model.model.layers.27.self_attn.o_proj.weight": "model.safetensors",
|
| 719 |
+
"language_model.model.layers.27.self_attn.q_proj.bias": "model.safetensors",
|
| 720 |
+
"language_model.model.layers.27.self_attn.q_proj.weight": "model.safetensors",
|
| 721 |
+
"language_model.model.layers.27.self_attn.v_proj.bias": "model.safetensors",
|
| 722 |
+
"language_model.model.layers.27.self_attn.v_proj.weight": "model.safetensors",
|
| 723 |
+
"language_model.model.layers.28.ada_rms_norm.linear1.weight": "model.safetensors",
|
| 724 |
+
"language_model.model.layers.28.ada_rms_norm.linear2.weight": "model.safetensors",
|
| 725 |
+
"language_model.model.layers.28.input_layernorm.weight": "model.safetensors",
|
| 726 |
+
"language_model.model.layers.28.mlp.down_proj.weight": "model.safetensors",
|
| 727 |
+
"language_model.model.layers.28.mlp.gate_proj.weight": "model.safetensors",
|
| 728 |
+
"language_model.model.layers.28.mlp.up_proj.weight": "model.safetensors",
|
| 729 |
+
"language_model.model.layers.28.post_attention_layernorm.weight": "model.safetensors",
|
| 730 |
+
"language_model.model.layers.28.self_attn.k_proj.bias": "model.safetensors",
|
| 731 |
+
"language_model.model.layers.28.self_attn.k_proj.weight": "model.safetensors",
|
| 732 |
+
"language_model.model.layers.28.self_attn.o_proj.weight": "model.safetensors",
|
| 733 |
+
"language_model.model.layers.28.self_attn.q_proj.bias": "model.safetensors",
|
| 734 |
+
"language_model.model.layers.28.self_attn.q_proj.weight": "model.safetensors",
|
| 735 |
+
"language_model.model.layers.28.self_attn.v_proj.bias": "model.safetensors",
|
| 736 |
+
"language_model.model.layers.28.self_attn.v_proj.weight": "model.safetensors",
|
| 737 |
+
"language_model.model.layers.29.ada_rms_norm.linear1.weight": "model.safetensors",
|
| 738 |
+
"language_model.model.layers.29.ada_rms_norm.linear2.weight": "model.safetensors",
|
| 739 |
+
"language_model.model.layers.29.input_layernorm.weight": "model.safetensors",
|
| 740 |
+
"language_model.model.layers.29.mlp.down_proj.weight": "model.safetensors",
|
| 741 |
+
"language_model.model.layers.29.mlp.gate_proj.weight": "model.safetensors",
|
| 742 |
+
"language_model.model.layers.29.mlp.up_proj.weight": "model.safetensors",
|
| 743 |
+
"language_model.model.layers.29.post_attention_layernorm.weight": "model.safetensors",
|
| 744 |
+
"language_model.model.layers.29.self_attn.k_proj.bias": "model.safetensors",
|
| 745 |
+
"language_model.model.layers.29.self_attn.k_proj.weight": "model.safetensors",
|
| 746 |
+
"language_model.model.layers.29.self_attn.o_proj.weight": "model.safetensors",
|
| 747 |
+
"language_model.model.layers.29.self_attn.q_proj.bias": "model.safetensors",
|
| 748 |
+
"language_model.model.layers.29.self_attn.q_proj.weight": "model.safetensors",
|
| 749 |
+
"language_model.model.layers.29.self_attn.v_proj.bias": "model.safetensors",
|
| 750 |
+
"language_model.model.layers.29.self_attn.v_proj.weight": "model.safetensors",
|
| 751 |
+
"language_model.model.layers.3.ada_rms_norm.linear1.weight": "model.safetensors",
|
| 752 |
+
"language_model.model.layers.3.ada_rms_norm.linear2.weight": "model.safetensors",
|
| 753 |
+
"language_model.model.layers.3.input_layernorm.weight": "model.safetensors",
|
| 754 |
+
"language_model.model.layers.3.mlp.down_proj.weight": "model.safetensors",
|
| 755 |
+
"language_model.model.layers.3.mlp.gate_proj.weight": "model.safetensors",
|
| 756 |
+
"language_model.model.layers.3.mlp.up_proj.weight": "model.safetensors",
|
| 757 |
+
"language_model.model.layers.3.post_attention_layernorm.weight": "model.safetensors",
|
| 758 |
+
"language_model.model.layers.3.self_attn.k_proj.bias": "model.safetensors",
|
| 759 |
+
"language_model.model.layers.3.self_attn.k_proj.weight": "model.safetensors",
|
| 760 |
+
"language_model.model.layers.3.self_attn.o_proj.weight": "model.safetensors",
|
| 761 |
+
"language_model.model.layers.3.self_attn.q_proj.bias": "model.safetensors",
|
| 762 |
+
"language_model.model.layers.3.self_attn.q_proj.weight": "model.safetensors",
|
| 763 |
+
"language_model.model.layers.3.self_attn.v_proj.bias": "model.safetensors",
|
| 764 |
+
"language_model.model.layers.3.self_attn.v_proj.weight": "model.safetensors",
|
| 765 |
+
"language_model.model.layers.30.ada_rms_norm.linear1.weight": "model.safetensors",
|
| 766 |
+
"language_model.model.layers.30.ada_rms_norm.linear2.weight": "model.safetensors",
|
| 767 |
+
"language_model.model.layers.30.input_layernorm.weight": "model.safetensors",
|
| 768 |
+
"language_model.model.layers.30.mlp.down_proj.weight": "model.safetensors",
|
| 769 |
+
"language_model.model.layers.30.mlp.gate_proj.weight": "model.safetensors",
|
| 770 |
+
"language_model.model.layers.30.mlp.up_proj.weight": "model.safetensors",
|
| 771 |
+
"language_model.model.layers.30.post_attention_layernorm.weight": "model.safetensors",
|
| 772 |
+
"language_model.model.layers.30.self_attn.k_proj.bias": "model.safetensors",
|
| 773 |
+
"language_model.model.layers.30.self_attn.k_proj.weight": "model.safetensors",
|
| 774 |
+
"language_model.model.layers.30.self_attn.o_proj.weight": "model.safetensors",
|
| 775 |
+
"language_model.model.layers.30.self_attn.q_proj.bias": "model.safetensors",
|
| 776 |
+
"language_model.model.layers.30.self_attn.q_proj.weight": "model.safetensors",
|
| 777 |
+
"language_model.model.layers.30.self_attn.v_proj.bias": "model.safetensors",
|
| 778 |
+
"language_model.model.layers.30.self_attn.v_proj.weight": "model.safetensors",
|
| 779 |
+
"language_model.model.layers.31.ada_rms_norm.linear1.weight": "model.safetensors",
|
| 780 |
+
"language_model.model.layers.31.ada_rms_norm.linear2.weight": "model.safetensors",
|
| 781 |
+
"language_model.model.layers.31.input_layernorm.weight": "model.safetensors",
|
| 782 |
+
"language_model.model.layers.31.mlp.down_proj.weight": "model.safetensors",
|
| 783 |
+
"language_model.model.layers.31.mlp.gate_proj.weight": "model.safetensors",
|
| 784 |
+
"language_model.model.layers.31.mlp.up_proj.weight": "model.safetensors",
|
| 785 |
+
"language_model.model.layers.31.post_attention_layernorm.weight": "model.safetensors",
|
| 786 |
+
"language_model.model.layers.31.self_attn.k_proj.bias": "model.safetensors",
|
| 787 |
+
"language_model.model.layers.31.self_attn.k_proj.weight": "model.safetensors",
|
| 788 |
+
"language_model.model.layers.31.self_attn.o_proj.weight": "model.safetensors",
|
| 789 |
+
"language_model.model.layers.31.self_attn.q_proj.bias": "model.safetensors",
|
| 790 |
+
"language_model.model.layers.31.self_attn.q_proj.weight": "model.safetensors",
|
| 791 |
+
"language_model.model.layers.31.self_attn.v_proj.bias": "model.safetensors",
|
| 792 |
+
"language_model.model.layers.31.self_attn.v_proj.weight": "model.safetensors",
|
| 793 |
+
"language_model.model.layers.32.ada_rms_norm.linear1.weight": "model.safetensors",
|
| 794 |
+
"language_model.model.layers.32.ada_rms_norm.linear2.weight": "model.safetensors",
|
| 795 |
+
"language_model.model.layers.32.input_layernorm.weight": "model.safetensors",
|
| 796 |
+
"language_model.model.layers.32.mlp.down_proj.weight": "model.safetensors",
|
| 797 |
+
"language_model.model.layers.32.mlp.gate_proj.weight": "model.safetensors",
|
| 798 |
+
"language_model.model.layers.32.mlp.up_proj.weight": "model.safetensors",
|
| 799 |
+
"language_model.model.layers.32.post_attention_layernorm.weight": "model.safetensors",
|
| 800 |
+
"language_model.model.layers.32.self_attn.k_proj.bias": "model.safetensors",
|
| 801 |
+
"language_model.model.layers.32.self_attn.k_proj.weight": "model.safetensors",
|
| 802 |
+
"language_model.model.layers.32.self_attn.o_proj.weight": "model.safetensors",
|
| 803 |
+
"language_model.model.layers.32.self_attn.q_proj.bias": "model.safetensors",
|
| 804 |
+
"language_model.model.layers.32.self_attn.q_proj.weight": "model.safetensors",
|
| 805 |
+
"language_model.model.layers.32.self_attn.v_proj.bias": "model.safetensors",
|
| 806 |
+
"language_model.model.layers.32.self_attn.v_proj.weight": "model.safetensors",
|
| 807 |
+
"language_model.model.layers.33.ada_rms_norm.linear1.weight": "model.safetensors",
|
| 808 |
+
"language_model.model.layers.33.ada_rms_norm.linear2.weight": "model.safetensors",
|
| 809 |
+
"language_model.model.layers.33.input_layernorm.weight": "model.safetensors",
|
| 810 |
+
"language_model.model.layers.33.mlp.down_proj.weight": "model.safetensors",
|
| 811 |
+
"language_model.model.layers.33.mlp.gate_proj.weight": "model.safetensors",
|
| 812 |
+
"language_model.model.layers.33.mlp.up_proj.weight": "model.safetensors",
|
| 813 |
+
"language_model.model.layers.33.post_attention_layernorm.weight": "model.safetensors",
|
| 814 |
+
"language_model.model.layers.33.self_attn.k_proj.bias": "model.safetensors",
|
| 815 |
+
"language_model.model.layers.33.self_attn.k_proj.weight": "model.safetensors",
|
| 816 |
+
"language_model.model.layers.33.self_attn.o_proj.weight": "model.safetensors",
|
| 817 |
+
"language_model.model.layers.33.self_attn.q_proj.bias": "model.safetensors",
|
| 818 |
+
"language_model.model.layers.33.self_attn.q_proj.weight": "model.safetensors",
|
| 819 |
+
"language_model.model.layers.33.self_attn.v_proj.bias": "model.safetensors",
|
| 820 |
+
"language_model.model.layers.33.self_attn.v_proj.weight": "model.safetensors",
|
| 821 |
+
"language_model.model.layers.34.ada_rms_norm.linear1.weight": "model.safetensors",
|
| 822 |
+
"language_model.model.layers.34.ada_rms_norm.linear2.weight": "model.safetensors",
|
| 823 |
+
"language_model.model.layers.34.input_layernorm.weight": "model.safetensors",
|
| 824 |
+
"language_model.model.layers.34.mlp.down_proj.weight": "model.safetensors",
|
| 825 |
+
"language_model.model.layers.34.mlp.gate_proj.weight": "model.safetensors",
|
| 826 |
+
"language_model.model.layers.34.mlp.up_proj.weight": "model.safetensors",
|
| 827 |
+
"language_model.model.layers.34.post_attention_layernorm.weight": "model.safetensors",
|
| 828 |
+
"language_model.model.layers.34.self_attn.k_proj.bias": "model.safetensors",
|
| 829 |
+
"language_model.model.layers.34.self_attn.k_proj.weight": "model.safetensors",
|
| 830 |
+
"language_model.model.layers.34.self_attn.o_proj.weight": "model.safetensors",
|
| 831 |
+
"language_model.model.layers.34.self_attn.q_proj.bias": "model.safetensors",
|
| 832 |
+
"language_model.model.layers.34.self_attn.q_proj.weight": "model.safetensors",
|
| 833 |
+
"language_model.model.layers.34.self_attn.v_proj.bias": "model.safetensors",
|
| 834 |
+
"language_model.model.layers.34.self_attn.v_proj.weight": "model.safetensors",
|
| 835 |
+
"language_model.model.layers.35.ada_rms_norm.linear1.weight": "model.safetensors",
|
| 836 |
+
"language_model.model.layers.35.ada_rms_norm.linear2.weight": "model.safetensors",
|
| 837 |
+
"language_model.model.layers.35.input_layernorm.weight": "model.safetensors",
|
| 838 |
+
"language_model.model.layers.35.mlp.down_proj.weight": "model.safetensors",
|
| 839 |
+
"language_model.model.layers.35.mlp.gate_proj.weight": "model.safetensors",
|
| 840 |
+
"language_model.model.layers.35.mlp.up_proj.weight": "model.safetensors",
|
| 841 |
+
"language_model.model.layers.35.post_attention_layernorm.weight": "model.safetensors",
|
| 842 |
+
"language_model.model.layers.35.self_attn.k_proj.bias": "model.safetensors",
|
| 843 |
+
"language_model.model.layers.35.self_attn.k_proj.weight": "model.safetensors",
|
| 844 |
+
"language_model.model.layers.35.self_attn.o_proj.weight": "model.safetensors",
|
| 845 |
+
"language_model.model.layers.35.self_attn.q_proj.bias": "model.safetensors",
|
| 846 |
+
"language_model.model.layers.35.self_attn.q_proj.weight": "model.safetensors",
|
| 847 |
+
"language_model.model.layers.35.self_attn.v_proj.bias": "model.safetensors",
|
| 848 |
+
"language_model.model.layers.35.self_attn.v_proj.weight": "model.safetensors",
|
| 849 |
+
"language_model.model.layers.4.ada_rms_norm.linear1.weight": "model.safetensors",
|
| 850 |
+
"language_model.model.layers.4.ada_rms_norm.linear2.weight": "model.safetensors",
|
| 851 |
+
"language_model.model.layers.4.input_layernorm.weight": "model.safetensors",
|
| 852 |
+
"language_model.model.layers.4.mlp.down_proj.weight": "model.safetensors",
|
| 853 |
+
"language_model.model.layers.4.mlp.gate_proj.weight": "model.safetensors",
|
| 854 |
+
"language_model.model.layers.4.mlp.up_proj.weight": "model.safetensors",
|
| 855 |
+
"language_model.model.layers.4.post_attention_layernorm.weight": "model.safetensors",
|
| 856 |
+
"language_model.model.layers.4.self_attn.k_proj.bias": "model.safetensors",
|
| 857 |
+
"language_model.model.layers.4.self_attn.k_proj.weight": "model.safetensors",
|
| 858 |
+
"language_model.model.layers.4.self_attn.o_proj.weight": "model.safetensors",
|
| 859 |
+
"language_model.model.layers.4.self_attn.q_proj.bias": "model.safetensors",
|
| 860 |
+
"language_model.model.layers.4.self_attn.q_proj.weight": "model.safetensors",
|
| 861 |
+
"language_model.model.layers.4.self_attn.v_proj.bias": "model.safetensors",
|
| 862 |
+
"language_model.model.layers.4.self_attn.v_proj.weight": "model.safetensors",
|
| 863 |
+
"language_model.model.layers.5.ada_rms_norm.linear1.weight": "model.safetensors",
|
| 864 |
+
"language_model.model.layers.5.ada_rms_norm.linear2.weight": "model.safetensors",
|
| 865 |
+
"language_model.model.layers.5.input_layernorm.weight": "model.safetensors",
|
| 866 |
+
"language_model.model.layers.5.mlp.down_proj.weight": "model.safetensors",
|
| 867 |
+
"language_model.model.layers.5.mlp.gate_proj.weight": "model.safetensors",
|
| 868 |
+
"language_model.model.layers.5.mlp.up_proj.weight": "model.safetensors",
|
| 869 |
+
"language_model.model.layers.5.post_attention_layernorm.weight": "model.safetensors",
|
| 870 |
+
"language_model.model.layers.5.self_attn.k_proj.bias": "model.safetensors",
|
| 871 |
+
"language_model.model.layers.5.self_attn.k_proj.weight": "model.safetensors",
|
| 872 |
+
"language_model.model.layers.5.self_attn.o_proj.weight": "model.safetensors",
|
| 873 |
+
"language_model.model.layers.5.self_attn.q_proj.bias": "model.safetensors",
|
| 874 |
+
"language_model.model.layers.5.self_attn.q_proj.weight": "model.safetensors",
|
| 875 |
+
"language_model.model.layers.5.self_attn.v_proj.bias": "model.safetensors",
|
| 876 |
+
"language_model.model.layers.5.self_attn.v_proj.weight": "model.safetensors",
|
| 877 |
+
"language_model.model.layers.6.ada_rms_norm.linear1.weight": "model.safetensors",
|
| 878 |
+
"language_model.model.layers.6.ada_rms_norm.linear2.weight": "model.safetensors",
|
| 879 |
+
"language_model.model.layers.6.input_layernorm.weight": "model.safetensors",
|
| 880 |
+
"language_model.model.layers.6.mlp.down_proj.weight": "model.safetensors",
|
| 881 |
+
"language_model.model.layers.6.mlp.gate_proj.weight": "model.safetensors",
|
| 882 |
+
"language_model.model.layers.6.mlp.up_proj.weight": "model.safetensors",
|
| 883 |
+
"language_model.model.layers.6.post_attention_layernorm.weight": "model.safetensors",
|
| 884 |
+
"language_model.model.layers.6.self_attn.k_proj.bias": "model.safetensors",
|
| 885 |
+
"language_model.model.layers.6.self_attn.k_proj.weight": "model.safetensors",
|
| 886 |
+
"language_model.model.layers.6.self_attn.o_proj.weight": "model.safetensors",
|
| 887 |
+
"language_model.model.layers.6.self_attn.q_proj.bias": "model.safetensors",
|
| 888 |
+
"language_model.model.layers.6.self_attn.q_proj.weight": "model.safetensors",
|
| 889 |
+
"language_model.model.layers.6.self_attn.v_proj.bias": "model.safetensors",
|
| 890 |
+
"language_model.model.layers.6.self_attn.v_proj.weight": "model.safetensors",
|
| 891 |
+
"language_model.model.layers.7.ada_rms_norm.linear1.weight": "model.safetensors",
|
| 892 |
+
"language_model.model.layers.7.ada_rms_norm.linear2.weight": "model.safetensors",
|
| 893 |
+
"language_model.model.layers.7.input_layernorm.weight": "model.safetensors",
|
| 894 |
+
"language_model.model.layers.7.mlp.down_proj.weight": "model.safetensors",
|
| 895 |
+
"language_model.model.layers.7.mlp.gate_proj.weight": "model.safetensors",
|
| 896 |
+
"language_model.model.layers.7.mlp.up_proj.weight": "model.safetensors",
|
| 897 |
+
"language_model.model.layers.7.post_attention_layernorm.weight": "model.safetensors",
|
| 898 |
+
"language_model.model.layers.7.self_attn.k_proj.bias": "model.safetensors",
|
| 899 |
+
"language_model.model.layers.7.self_attn.k_proj.weight": "model.safetensors",
|
| 900 |
+
"language_model.model.layers.7.self_attn.o_proj.weight": "model.safetensors",
|
| 901 |
+
"language_model.model.layers.7.self_attn.q_proj.bias": "model.safetensors",
|
| 902 |
+
"language_model.model.layers.7.self_attn.q_proj.weight": "model.safetensors",
|
| 903 |
+
"language_model.model.layers.7.self_attn.v_proj.bias": "model.safetensors",
|
| 904 |
+
"language_model.model.layers.7.self_attn.v_proj.weight": "model.safetensors",
|
| 905 |
+
"language_model.model.layers.8.ada_rms_norm.linear1.weight": "model.safetensors",
|
| 906 |
+
"language_model.model.layers.8.ada_rms_norm.linear2.weight": "model.safetensors",
|
| 907 |
+
"language_model.model.layers.8.input_layernorm.weight": "model.safetensors",
|
| 908 |
+
"language_model.model.layers.8.mlp.down_proj.weight": "model.safetensors",
|
| 909 |
+
"language_model.model.layers.8.mlp.gate_proj.weight": "model.safetensors",
|
| 910 |
+
"language_model.model.layers.8.mlp.up_proj.weight": "model.safetensors",
|
| 911 |
+
"language_model.model.layers.8.post_attention_layernorm.weight": "model.safetensors",
|
| 912 |
+
"language_model.model.layers.8.self_attn.k_proj.bias": "model.safetensors",
|
| 913 |
+
"language_model.model.layers.8.self_attn.k_proj.weight": "model.safetensors",
|
| 914 |
+
"language_model.model.layers.8.self_attn.o_proj.weight": "model.safetensors",
|
| 915 |
+
"language_model.model.layers.8.self_attn.q_proj.bias": "model.safetensors",
|
| 916 |
+
"language_model.model.layers.8.self_attn.q_proj.weight": "model.safetensors",
|
| 917 |
+
"language_model.model.layers.8.self_attn.v_proj.bias": "model.safetensors",
|
| 918 |
+
"language_model.model.layers.8.self_attn.v_proj.weight": "model.safetensors",
|
| 919 |
+
"language_model.model.layers.9.ada_rms_norm.linear1.weight": "model.safetensors",
|
| 920 |
+
"language_model.model.layers.9.ada_rms_norm.linear2.weight": "model.safetensors",
|
| 921 |
+
"language_model.model.layers.9.input_layernorm.weight": "model.safetensors",
|
| 922 |
+
"language_model.model.layers.9.mlp.down_proj.weight": "model.safetensors",
|
| 923 |
+
"language_model.model.layers.9.mlp.gate_proj.weight": "model.safetensors",
|
| 924 |
+
"language_model.model.layers.9.mlp.up_proj.weight": "model.safetensors",
|
| 925 |
+
"language_model.model.layers.9.post_attention_layernorm.weight": "model.safetensors",
|
| 926 |
+
"language_model.model.layers.9.self_attn.k_proj.bias": "model.safetensors",
|
| 927 |
+
"language_model.model.layers.9.self_attn.k_proj.weight": "model.safetensors",
|
| 928 |
+
"language_model.model.layers.9.self_attn.o_proj.weight": "model.safetensors",
|
| 929 |
+
"language_model.model.layers.9.self_attn.q_proj.bias": "model.safetensors",
|
| 930 |
+
"language_model.model.layers.9.self_attn.q_proj.weight": "model.safetensors",
|
| 931 |
+
"language_model.model.layers.9.self_attn.v_proj.bias": "model.safetensors",
|
| 932 |
+
"language_model.model.layers.9.self_attn.v_proj.weight": "model.safetensors",
|
| 933 |
+
"language_model.model.norm.weight": "model.safetensors",
|
| 934 |
+
"multi_modal_projector.linear_1.weight": "model.safetensors",
|
| 935 |
+
"multi_modal_projector.linear_2.weight": "model.safetensors",
|
| 936 |
+
"semantic_vad_heads.0.bias": "semantic_vad_heads.safetensors",
|
| 937 |
+
"semantic_vad_heads.0.weight": "semantic_vad_heads.safetensors",
|
| 938 |
+
"semantic_vad_heads.1.bias": "semantic_vad_heads.safetensors",
|
| 939 |
+
"semantic_vad_heads.1.weight": "semantic_vad_heads.safetensors",
|
| 940 |
+
"semantic_vad_heads.2.bias": "semantic_vad_heads.safetensors",
|
| 941 |
+
"semantic_vad_heads.2.weight": "semantic_vad_heads.safetensors",
|
| 942 |
+
"semantic_vad_heads.3.bias": "semantic_vad_heads.safetensors",
|
| 943 |
+
"semantic_vad_heads.3.weight": "semantic_vad_heads.safetensors"
|
| 944 |
+
}
|
| 945 |
+
}
|
modeling_audio8_asr_infinite.py
ADDED
|
@@ -0,0 +1,1338 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Audio8 ASR Infinite with Voxtral Realtime delay conditioning and a Qwen decoder.
|
| 2 |
+
|
| 3 |
+
This model intentionally lives next to, not on top of, the existing
|
| 4 |
+
legacy ``audio8_streaming_asr`` implementation. The audio tower,
|
| 5 |
+
projector, tokenizer contract, and Qwen decoder/head are preserved. The delay
|
| 6 |
+
path is changed from Audio8 ASR Infinite's input-level learned ``delay_embedding`` to the
|
| 7 |
+
Voxtral Realtime mechanism:
|
| 8 |
+
|
| 9 |
+
``num_delay_tokens -> sinusoidal time embedding -> per-layer adaptive MLP -> post-attention hidden scaling``.
|
| 10 |
+
"""
|
| 11 |
+
|
| 12 |
+
from __future__ import annotations
|
| 13 |
+
|
| 14 |
+
from types import GeneratorType
|
| 15 |
+
from typing import Any, Sequence
|
| 16 |
+
|
| 17 |
+
import torch
|
| 18 |
+
from torch import nn
|
| 19 |
+
from torch.nn import functional as F
|
| 20 |
+
from transformers import (
|
| 21 |
+
AutoModel,
|
| 22 |
+
PreTrainedModel,
|
| 23 |
+
Qwen2Config,
|
| 24 |
+
Qwen3Config,
|
| 25 |
+
)
|
| 26 |
+
from transformers.generation import GenerationMixin
|
| 27 |
+
from transformers.activations import ACT2FN
|
| 28 |
+
from transformers.modeling_outputs import CausalLMOutputWithPast
|
| 29 |
+
from transformers.models.qwen2.modeling_qwen2 import (
|
| 30 |
+
Qwen2DecoderLayer,
|
| 31 |
+
Qwen2ForCausalLM,
|
| 32 |
+
Qwen2Model,
|
| 33 |
+
Qwen2PreTrainedModel,
|
| 34 |
+
Qwen2RMSNorm,
|
| 35 |
+
Qwen2RotaryEmbedding,
|
| 36 |
+
)
|
| 37 |
+
from transformers.models.qwen3.modeling_qwen3 import (
|
| 38 |
+
Qwen3DecoderLayer,
|
| 39 |
+
Qwen3ForCausalLM,
|
| 40 |
+
Qwen3Model,
|
| 41 |
+
Qwen3PreTrainedModel,
|
| 42 |
+
Qwen3RMSNorm,
|
| 43 |
+
Qwen3RotaryEmbedding,
|
| 44 |
+
)
|
| 45 |
+
from transformers.models.voxtral_realtime import modeling_voxtral_realtime as _voxtral_realtime_modeling
|
| 46 |
+
|
| 47 |
+
from .configuration_audio8_asr_infinite import (
|
| 48 |
+
AUDIO8_ASR_INFINITE_WEIGHT_FORMAT_VERSION,
|
| 49 |
+
DEFAULT_SEMANTIC_VAD_NUM_CLASSES,
|
| 50 |
+
Audio8ASRInfiniteConfig,
|
| 51 |
+
)
|
| 52 |
+
|
| 53 |
+
VoxtralRealtimeTextAdaRmsNorm = _voxtral_realtime_modeling.VoxtralRealtimeTextAdaRmsNorm
|
| 54 |
+
VoxtralRealtimeTimeEmbedding = _voxtral_realtime_modeling.VoxtralRealtimeTimeEmbedding
|
| 55 |
+
|
| 56 |
+
# The model's Qwen tokenizer special-token contract: the five core special ids
|
| 57 |
+
# consumed by the simulated-streaming decoder, plus the streaming / language
|
| 58 |
+
# special-token strings.
|
| 59 |
+
STREAMING_PAD_TOKEN = "[STREAMING_PAD]"
|
| 60 |
+
STREAMING_WORD_TOKEN = "[STREAMING_WORD]"
|
| 61 |
+
LANGUAGE_ZH_TOKEN = "[LANGUAGE_ZH]"
|
| 62 |
+
LANGUAGE_EN_TOKEN = "[LANGUAGE_EN]"
|
| 63 |
+
QWEN_AUDIO_PAD_TOKEN = "<|audio_pad|>"
|
| 64 |
+
QWEN_ASR_TEXT_TOKEN = "<asr_text>"
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def resolve_token_id(tokenizer: Any, *, attr_name: str | None, token: str) -> int:
|
| 68 |
+
if attr_name:
|
| 69 |
+
token_id = getattr(tokenizer, attr_name, None)
|
| 70 |
+
if token_id is not None:
|
| 71 |
+
return int(token_id)
|
| 72 |
+
token_id = tokenizer.convert_tokens_to_ids(token)
|
| 73 |
+
if token_id is not None and int(token_id) >= 0:
|
| 74 |
+
return int(token_id)
|
| 75 |
+
raise ValueError(f"Tokenizer cannot resolve required token id for {token!r}.")
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
def resolve_qwen_streaming_special_token_ids(tokenizer: Any) -> dict[str, int]:
|
| 79 |
+
bos_token_id = resolve_token_id(tokenizer, attr_name="bos_token_id", token="<|im_start|>")
|
| 80 |
+
eos_token_id = resolve_token_id(tokenizer, attr_name="eos_token_id", token="<|im_end|>")
|
| 81 |
+
pad_token_id = resolve_token_id(tokenizer, attr_name="pad_token_id", token="<|endoftext|>")
|
| 82 |
+
streaming_pad_token_id = resolve_token_id(tokenizer, attr_name=None, token=STREAMING_PAD_TOKEN)
|
| 83 |
+
streaming_word_token_id = resolve_token_id(tokenizer, attr_name=None, token=STREAMING_WORD_TOKEN)
|
| 84 |
+
return {
|
| 85 |
+
"bos_token_id": bos_token_id,
|
| 86 |
+
"eos_token_id": eos_token_id,
|
| 87 |
+
"pad_token_id": pad_token_id,
|
| 88 |
+
"streaming_pad_token_id": streaming_pad_token_id,
|
| 89 |
+
"streaming_word_token_id": streaming_word_token_id,
|
| 90 |
+
}
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def ensure_voxtral_streaming_tokens(tokenizer: Any) -> int:
|
| 94 |
+
added = tokenizer.add_special_tokens(
|
| 95 |
+
{
|
| 96 |
+
"additional_special_tokens": [
|
| 97 |
+
token
|
| 98 |
+
for token in (
|
| 99 |
+
STREAMING_PAD_TOKEN,
|
| 100 |
+
STREAMING_WORD_TOKEN,
|
| 101 |
+
LANGUAGE_ZH_TOKEN,
|
| 102 |
+
LANGUAGE_EN_TOKEN,
|
| 103 |
+
)
|
| 104 |
+
if tokenizer.convert_tokens_to_ids(token) is None
|
| 105 |
+
or int(tokenizer.convert_tokens_to_ids(token)) < 0
|
| 106 |
+
]
|
| 107 |
+
}
|
| 108 |
+
)
|
| 109 |
+
resolve_qwen_streaming_special_token_ids(tokenizer)
|
| 110 |
+
return int(added)
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def resolve_qwen_language_token_id(
|
| 115 |
+
tokenizer: Any,
|
| 116 |
+
language: str,
|
| 117 |
+
) -> int:
|
| 118 |
+
"""把规范语言字段映射到对应的 prompt token。"""
|
| 119 |
+
|
| 120 |
+
token_by_language = {
|
| 121 |
+
"zh": LANGUAGE_ZH_TOKEN,
|
| 122 |
+
"en": LANGUAGE_EN_TOKEN,
|
| 123 |
+
}
|
| 124 |
+
normalized = str(language).strip().lower()
|
| 125 |
+
token = token_by_language.get(normalized)
|
| 126 |
+
if token is None:
|
| 127 |
+
raise ValueError(
|
| 128 |
+
"language must be exactly 'zh' or 'en', "
|
| 129 |
+
f"got {language!r}."
|
| 130 |
+
)
|
| 131 |
+
return resolve_token_id(
|
| 132 |
+
tokenizer,
|
| 133 |
+
attr_name=None,
|
| 134 |
+
token=token,
|
| 135 |
+
)
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
class Audio8ASRInfiniteMaxFrameLenProjector(nn.Module):
|
| 139 |
+
"""Project audio groups padded to the configured maximum frame length."""
|
| 140 |
+
|
| 141 |
+
def __init__(self, config: Audio8ASRInfiniteConfig) -> None:
|
| 142 |
+
super().__init__()
|
| 143 |
+
self.linear_1 = nn.Linear(
|
| 144 |
+
config.projection_size,
|
| 145 |
+
config.text_config.hidden_size,
|
| 146 |
+
bias=False,
|
| 147 |
+
)
|
| 148 |
+
self.act = ACT2FN[config.projector_hidden_act]
|
| 149 |
+
self.linear_2 = nn.Linear(
|
| 150 |
+
config.text_config.hidden_size,
|
| 151 |
+
config.text_config.hidden_size,
|
| 152 |
+
bias=False,
|
| 153 |
+
)
|
| 154 |
+
|
| 155 |
+
def forward(self, audio_features: torch.Tensor) -> torch.Tensor:
|
| 156 |
+
hidden_states = self.linear_1(audio_features)
|
| 157 |
+
hidden_states = self.act(hidden_states)
|
| 158 |
+
return self.linear_2(hidden_states)
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
class Qwen3RealtimeV1DecoderLayer(Qwen3DecoderLayer):
|
| 162 |
+
"""Qwen3 decoder layer with Voxtral-style delay modulation before the MLP."""
|
| 163 |
+
|
| 164 |
+
def __init__(self, config: Qwen3Config, layer_idx: int):
|
| 165 |
+
super().__init__(config, layer_idx)
|
| 166 |
+
self.ada_rms_norm = VoxtralRealtimeTextAdaRmsNorm(config)
|
| 167 |
+
|
| 168 |
+
def forward(
|
| 169 |
+
self,
|
| 170 |
+
hidden_states: torch.Tensor,
|
| 171 |
+
attention_mask: torch.Tensor | None = None,
|
| 172 |
+
position_ids: torch.LongTensor | None = None,
|
| 173 |
+
past_key_values: Any | None = None,
|
| 174 |
+
use_cache: bool | None = False,
|
| 175 |
+
position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None,
|
| 176 |
+
t_cond: torch.Tensor | None = None,
|
| 177 |
+
**kwargs: Any,
|
| 178 |
+
) -> torch.Tensor:
|
| 179 |
+
residual = hidden_states
|
| 180 |
+
hidden_states = self.input_layernorm(hidden_states)
|
| 181 |
+
hidden_states, _ = self.self_attn(
|
| 182 |
+
hidden_states=hidden_states,
|
| 183 |
+
attention_mask=attention_mask,
|
| 184 |
+
position_ids=position_ids,
|
| 185 |
+
past_key_values=past_key_values,
|
| 186 |
+
use_cache=use_cache,
|
| 187 |
+
position_embeddings=position_embeddings,
|
| 188 |
+
**kwargs,
|
| 189 |
+
)
|
| 190 |
+
hidden_states = residual + hidden_states
|
| 191 |
+
|
| 192 |
+
residual = hidden_states
|
| 193 |
+
hidden_states = self.post_attention_layernorm(hidden_states)
|
| 194 |
+
if t_cond is None:
|
| 195 |
+
raise ValueError("Qwen3RealtimeV1DecoderLayer requires `t_cond`.")
|
| 196 |
+
hidden_states = hidden_states * (1 + self.ada_rms_norm(t_cond).to(dtype=hidden_states.dtype))
|
| 197 |
+
hidden_states = self.mlp(hidden_states)
|
| 198 |
+
hidden_states = residual + hidden_states
|
| 199 |
+
return hidden_states
|
| 200 |
+
|
| 201 |
+
|
| 202 |
+
class Qwen2RealtimeV1DecoderLayer(Qwen2DecoderLayer):
|
| 203 |
+
"""Qwen2 decoder layer with Voxtral-style delay modulation before the MLP."""
|
| 204 |
+
|
| 205 |
+
def __init__(self, config: Qwen2Config, layer_idx: int):
|
| 206 |
+
super().__init__(config, layer_idx)
|
| 207 |
+
self.ada_rms_norm = VoxtralRealtimeTextAdaRmsNorm(config)
|
| 208 |
+
|
| 209 |
+
def forward(
|
| 210 |
+
self,
|
| 211 |
+
hidden_states: torch.Tensor,
|
| 212 |
+
attention_mask: torch.Tensor | None = None,
|
| 213 |
+
position_ids: torch.LongTensor | None = None,
|
| 214 |
+
past_key_values: Any | None = None,
|
| 215 |
+
use_cache: bool | None = False,
|
| 216 |
+
position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None,
|
| 217 |
+
t_cond: torch.Tensor | None = None,
|
| 218 |
+
**kwargs: Any,
|
| 219 |
+
) -> torch.Tensor:
|
| 220 |
+
residual = hidden_states
|
| 221 |
+
hidden_states = self.input_layernorm(hidden_states)
|
| 222 |
+
hidden_states, _ = self.self_attn(
|
| 223 |
+
hidden_states=hidden_states,
|
| 224 |
+
attention_mask=attention_mask,
|
| 225 |
+
position_ids=position_ids,
|
| 226 |
+
past_key_values=past_key_values,
|
| 227 |
+
use_cache=use_cache,
|
| 228 |
+
position_embeddings=position_embeddings,
|
| 229 |
+
**kwargs,
|
| 230 |
+
)
|
| 231 |
+
hidden_states = residual + hidden_states
|
| 232 |
+
|
| 233 |
+
residual = hidden_states
|
| 234 |
+
hidden_states = self.post_attention_layernorm(hidden_states)
|
| 235 |
+
if t_cond is None:
|
| 236 |
+
raise ValueError("Qwen2RealtimeV1DecoderLayer requires `t_cond`.")
|
| 237 |
+
hidden_states = hidden_states * (
|
| 238 |
+
1
|
| 239 |
+
+ self.ada_rms_norm(t_cond).to(
|
| 240 |
+
dtype=hidden_states.dtype
|
| 241 |
+
)
|
| 242 |
+
)
|
| 243 |
+
hidden_states = self.mlp(hidden_states)
|
| 244 |
+
hidden_states = residual + hidden_states
|
| 245 |
+
return hidden_states
|
| 246 |
+
|
| 247 |
+
|
| 248 |
+
class Audio8ASRInfiniteQwen2TextModel(Qwen2Model):
|
| 249 |
+
"""Qwen2 text backbone built directly from Realtime V1 decoder layers."""
|
| 250 |
+
|
| 251 |
+
_no_split_modules = ["Qwen2RealtimeV1DecoderLayer"]
|
| 252 |
+
|
| 253 |
+
def __init__(self, config: Qwen2Config) -> None:
|
| 254 |
+
Qwen2PreTrainedModel.__init__(self, config)
|
| 255 |
+
self.padding_idx = config.pad_token_id
|
| 256 |
+
self.vocab_size = config.vocab_size
|
| 257 |
+
self.embed_tokens = nn.Embedding(
|
| 258 |
+
config.vocab_size,
|
| 259 |
+
config.hidden_size,
|
| 260 |
+
self.padding_idx,
|
| 261 |
+
)
|
| 262 |
+
self.layers = nn.ModuleList(
|
| 263 |
+
[
|
| 264 |
+
Qwen2RealtimeV1DecoderLayer(config, layer_idx)
|
| 265 |
+
for layer_idx in range(config.num_hidden_layers)
|
| 266 |
+
]
|
| 267 |
+
)
|
| 268 |
+
self.norm = Qwen2RMSNorm(
|
| 269 |
+
config.hidden_size,
|
| 270 |
+
eps=config.rms_norm_eps,
|
| 271 |
+
)
|
| 272 |
+
self.rotary_emb = Qwen2RotaryEmbedding(config=config)
|
| 273 |
+
self.gradient_checkpointing = False
|
| 274 |
+
self.has_sliding_layers = (
|
| 275 |
+
"sliding_attention" in self.config.layer_types
|
| 276 |
+
)
|
| 277 |
+
self.post_init()
|
| 278 |
+
|
| 279 |
+
def forward(
|
| 280 |
+
self,
|
| 281 |
+
input_ids: torch.LongTensor | None = None,
|
| 282 |
+
attention_mask: torch.Tensor | None = None,
|
| 283 |
+
position_ids: torch.LongTensor | None = None,
|
| 284 |
+
past_key_values: Any | None = None,
|
| 285 |
+
inputs_embeds: torch.FloatTensor | None = None,
|
| 286 |
+
use_cache: bool | None = None,
|
| 287 |
+
t_cond: torch.Tensor | None = None,
|
| 288 |
+
**kwargs: Any,
|
| 289 |
+
) -> Any:
|
| 290 |
+
if t_cond is None:
|
| 291 |
+
raise ValueError(
|
| 292 |
+
"Audio8ASRInfiniteQwen2TextModel requires `t_cond`."
|
| 293 |
+
)
|
| 294 |
+
return super().forward(
|
| 295 |
+
input_ids=input_ids,
|
| 296 |
+
attention_mask=attention_mask,
|
| 297 |
+
position_ids=position_ids,
|
| 298 |
+
past_key_values=past_key_values,
|
| 299 |
+
inputs_embeds=inputs_embeds,
|
| 300 |
+
use_cache=use_cache,
|
| 301 |
+
t_cond=t_cond,
|
| 302 |
+
**kwargs,
|
| 303 |
+
)
|
| 304 |
+
|
| 305 |
+
|
| 306 |
+
class Audio8ASRInfiniteQwen2ForCausalLM(Qwen2ForCausalLM):
|
| 307 |
+
"""Qwen2 causal LM built directly on the Realtime V1 text backbone."""
|
| 308 |
+
|
| 309 |
+
_no_split_modules = ["Qwen2RealtimeV1DecoderLayer"]
|
| 310 |
+
|
| 311 |
+
def __init__(self, config: Qwen2Config) -> None:
|
| 312 |
+
Qwen2PreTrainedModel.__init__(self, config)
|
| 313 |
+
self.model = Audio8ASRInfiniteQwen2TextModel(config)
|
| 314 |
+
self.vocab_size = config.vocab_size
|
| 315 |
+
self.lm_head = nn.Linear(
|
| 316 |
+
config.hidden_size,
|
| 317 |
+
config.vocab_size,
|
| 318 |
+
bias=False,
|
| 319 |
+
)
|
| 320 |
+
self.post_init()
|
| 321 |
+
|
| 322 |
+
def forward(
|
| 323 |
+
self,
|
| 324 |
+
input_ids: torch.LongTensor | None = None,
|
| 325 |
+
attention_mask: torch.Tensor | None = None,
|
| 326 |
+
position_ids: torch.LongTensor | None = None,
|
| 327 |
+
past_key_values: Any | None = None,
|
| 328 |
+
inputs_embeds: torch.FloatTensor | None = None,
|
| 329 |
+
use_cache: bool | None = None,
|
| 330 |
+
logits_to_keep: int | torch.Tensor = 0,
|
| 331 |
+
t_cond: torch.Tensor | None = None,
|
| 332 |
+
**kwargs: Any,
|
| 333 |
+
) -> CausalLMOutputWithPast:
|
| 334 |
+
if t_cond is None:
|
| 335 |
+
raise ValueError(
|
| 336 |
+
"Audio8ASRInfiniteQwen2ForCausalLM requires `t_cond`."
|
| 337 |
+
)
|
| 338 |
+
return super().forward(
|
| 339 |
+
input_ids=input_ids,
|
| 340 |
+
attention_mask=attention_mask,
|
| 341 |
+
position_ids=position_ids,
|
| 342 |
+
past_key_values=past_key_values,
|
| 343 |
+
inputs_embeds=inputs_embeds,
|
| 344 |
+
use_cache=use_cache,
|
| 345 |
+
logits_to_keep=logits_to_keep,
|
| 346 |
+
t_cond=t_cond,
|
| 347 |
+
**kwargs,
|
| 348 |
+
)
|
| 349 |
+
|
| 350 |
+
|
| 351 |
+
class Audio8ASRInfiniteTextModel(Qwen3Model):
|
| 352 |
+
"""Qwen3 text backbone built directly from Realtime V1 decoder layers."""
|
| 353 |
+
|
| 354 |
+
_no_split_modules = ["Qwen3RealtimeV1DecoderLayer"]
|
| 355 |
+
|
| 356 |
+
def __init__(self, config: Qwen3Config) -> None:
|
| 357 |
+
# Skip Qwen3Model.__init__: it would allocate vanilla decoder layers.
|
| 358 |
+
Qwen3PreTrainedModel.__init__(self, config)
|
| 359 |
+
self.padding_idx = config.pad_token_id
|
| 360 |
+
self.vocab_size = config.vocab_size
|
| 361 |
+
self.embed_tokens = nn.Embedding(
|
| 362 |
+
config.vocab_size,
|
| 363 |
+
config.hidden_size,
|
| 364 |
+
self.padding_idx,
|
| 365 |
+
)
|
| 366 |
+
self.layers = nn.ModuleList(
|
| 367 |
+
[
|
| 368 |
+
Qwen3RealtimeV1DecoderLayer(config, layer_idx)
|
| 369 |
+
for layer_idx in range(config.num_hidden_layers)
|
| 370 |
+
]
|
| 371 |
+
)
|
| 372 |
+
self.norm = Qwen3RMSNorm(
|
| 373 |
+
config.hidden_size,
|
| 374 |
+
eps=config.rms_norm_eps,
|
| 375 |
+
)
|
| 376 |
+
self.rotary_emb = Qwen3RotaryEmbedding(config=config)
|
| 377 |
+
self.gradient_checkpointing = False
|
| 378 |
+
self.has_sliding_layers = (
|
| 379 |
+
"sliding_attention" in self.config.layer_types
|
| 380 |
+
)
|
| 381 |
+
self.post_init()
|
| 382 |
+
|
| 383 |
+
def forward(
|
| 384 |
+
self,
|
| 385 |
+
input_ids: torch.LongTensor | None = None,
|
| 386 |
+
attention_mask: torch.Tensor | None = None,
|
| 387 |
+
position_ids: torch.LongTensor | None = None,
|
| 388 |
+
past_key_values: Any | None = None,
|
| 389 |
+
inputs_embeds: torch.FloatTensor | None = None,
|
| 390 |
+
use_cache: bool | None = None,
|
| 391 |
+
t_cond: torch.Tensor | None = None,
|
| 392 |
+
**kwargs: Any,
|
| 393 |
+
) -> Any:
|
| 394 |
+
if t_cond is None:
|
| 395 |
+
raise ValueError(
|
| 396 |
+
"Audio8ASRInfiniteTextModel requires `t_cond`."
|
| 397 |
+
)
|
| 398 |
+
return super().forward(
|
| 399 |
+
input_ids=input_ids,
|
| 400 |
+
attention_mask=attention_mask,
|
| 401 |
+
position_ids=position_ids,
|
| 402 |
+
past_key_values=past_key_values,
|
| 403 |
+
inputs_embeds=inputs_embeds,
|
| 404 |
+
use_cache=use_cache,
|
| 405 |
+
t_cond=t_cond,
|
| 406 |
+
**kwargs,
|
| 407 |
+
)
|
| 408 |
+
|
| 409 |
+
|
| 410 |
+
class Audio8ASRInfiniteForCausalLM(Qwen3ForCausalLM):
|
| 411 |
+
"""Qwen3 causal LM built directly on the Realtime V1 text backbone."""
|
| 412 |
+
|
| 413 |
+
_no_split_modules = ["Qwen3RealtimeV1DecoderLayer"]
|
| 414 |
+
|
| 415 |
+
def __init__(self, config: Qwen3Config) -> None:
|
| 416 |
+
# Skip Qwen3ForCausalLM.__init__: the backbone must be native V1.
|
| 417 |
+
Qwen3PreTrainedModel.__init__(self, config)
|
| 418 |
+
self.model = Audio8ASRInfiniteTextModel(config)
|
| 419 |
+
self.vocab_size = config.vocab_size
|
| 420 |
+
self.lm_head = nn.Linear(
|
| 421 |
+
config.hidden_size,
|
| 422 |
+
config.vocab_size,
|
| 423 |
+
bias=False,
|
| 424 |
+
)
|
| 425 |
+
self.post_init()
|
| 426 |
+
|
| 427 |
+
def forward(
|
| 428 |
+
self,
|
| 429 |
+
input_ids: torch.LongTensor | None = None,
|
| 430 |
+
attention_mask: torch.Tensor | None = None,
|
| 431 |
+
position_ids: torch.LongTensor | None = None,
|
| 432 |
+
past_key_values: Any | None = None,
|
| 433 |
+
inputs_embeds: torch.FloatTensor | None = None,
|
| 434 |
+
use_cache: bool | None = None,
|
| 435 |
+
logits_to_keep: int | torch.Tensor = 0,
|
| 436 |
+
t_cond: torch.Tensor | None = None,
|
| 437 |
+
**kwargs: Any,
|
| 438 |
+
) -> CausalLMOutputWithPast:
|
| 439 |
+
if t_cond is None:
|
| 440 |
+
raise ValueError(
|
| 441 |
+
"Audio8ASRInfiniteForCausalLM requires `t_cond`."
|
| 442 |
+
)
|
| 443 |
+
return super().forward(
|
| 444 |
+
input_ids=input_ids,
|
| 445 |
+
attention_mask=attention_mask,
|
| 446 |
+
position_ids=position_ids,
|
| 447 |
+
past_key_values=past_key_values,
|
| 448 |
+
inputs_embeds=inputs_embeds,
|
| 449 |
+
use_cache=use_cache,
|
| 450 |
+
logits_to_keep=logits_to_keep,
|
| 451 |
+
t_cond=t_cond,
|
| 452 |
+
**kwargs,
|
| 453 |
+
)
|
| 454 |
+
|
| 455 |
+
|
| 456 |
+
class Audio8ASRInfiniteForConditionalGeneration(PreTrainedModel, GenerationMixin):
|
| 457 |
+
"""Voxtral audio tower + Qwen decoder/head + Voxtral-style delay conditioning."""
|
| 458 |
+
|
| 459 |
+
config_class = Audio8ASRInfiniteConfig
|
| 460 |
+
base_model_prefix = "audio8_asr_infinite"
|
| 461 |
+
_tied_weights_keys = {
|
| 462 |
+
"language_model.lm_head.weight": (
|
| 463 |
+
"language_model.model.embed_tokens.weight"
|
| 464 |
+
),
|
| 465 |
+
}
|
| 466 |
+
_no_split_modules = [
|
| 467 |
+
"VoxtralRealtimeEncoderLayer",
|
| 468 |
+
"Qwen2RealtimeV1DecoderLayer",
|
| 469 |
+
"Qwen3RealtimeV1DecoderLayer",
|
| 470 |
+
]
|
| 471 |
+
supports_gradient_checkpointing = True
|
| 472 |
+
_supports_cache_class = True
|
| 473 |
+
_supports_sdpa = True
|
| 474 |
+
_supports_flash_attn = True
|
| 475 |
+
_supports_flex_attn = True
|
| 476 |
+
|
| 477 |
+
@classmethod
|
| 478 |
+
def from_pretrained(
|
| 479 |
+
cls,
|
| 480 |
+
pretrained_model_name_or_path: str | None,
|
| 481 |
+
*model_args: Any,
|
| 482 |
+
**kwargs: Any,
|
| 483 |
+
) -> Any:
|
| 484 |
+
if kwargs.get("ignore_mismatched_sizes", False):
|
| 485 |
+
raise ValueError(
|
| 486 |
+
"Audio8 ASR Infinite forbids `ignore_mismatched_sizes`; "
|
| 487 |
+
"convert the checkpoint to the exact current weight format."
|
| 488 |
+
)
|
| 489 |
+
return_loading_info = bool(
|
| 490 |
+
kwargs.pop("output_loading_info", False)
|
| 491 |
+
)
|
| 492 |
+
model, loading_info = super().from_pretrained(
|
| 493 |
+
pretrained_model_name_or_path,
|
| 494 |
+
*model_args,
|
| 495 |
+
output_loading_info=True,
|
| 496 |
+
**kwargs,
|
| 497 |
+
)
|
| 498 |
+
incompatible = {
|
| 499 |
+
name: loading_info.get(name)
|
| 500 |
+
for name in (
|
| 501 |
+
"missing_keys",
|
| 502 |
+
"unexpected_keys",
|
| 503 |
+
"mismatched_keys",
|
| 504 |
+
"error_msgs",
|
| 505 |
+
)
|
| 506 |
+
if loading_info.get(name)
|
| 507 |
+
}
|
| 508 |
+
if incompatible:
|
| 509 |
+
raise RuntimeError(
|
| 510 |
+
"Audio8 ASR Infinite checkpoint does not exactly match the "
|
| 511 |
+
"current weight format. Convert it before loading. "
|
| 512 |
+
f"incompatible={incompatible}"
|
| 513 |
+
)
|
| 514 |
+
loaded_time_embedding = model.time_embedding
|
| 515 |
+
time_embedding = VoxtralRealtimeTimeEmbedding(
|
| 516 |
+
model.config.text_config.hidden_size,
|
| 517 |
+
theta=float(getattr(loaded_time_embedding, "theta", 10000.0)),
|
| 518 |
+
)
|
| 519 |
+
loaded_buffer = loaded_time_embedding.inv_freq
|
| 520 |
+
if loaded_buffer.device.type != "meta":
|
| 521 |
+
time_embedding.to(device=loaded_buffer.device)
|
| 522 |
+
if not torch.isfinite(time_embedding.inv_freq).all():
|
| 523 |
+
raise RuntimeError(
|
| 524 |
+
"Voxtral time embedding initialization is non-finite."
|
| 525 |
+
)
|
| 526 |
+
model.time_embedding = time_embedding
|
| 527 |
+
if return_loading_info:
|
| 528 |
+
return model, loading_info
|
| 529 |
+
return model
|
| 530 |
+
|
| 531 |
+
def __init__(self, config: Audio8ASRInfiniteConfig) -> None:
|
| 532 |
+
super().__init__(config)
|
| 533 |
+
self.vocab_size = config.text_config.vocab_size
|
| 534 |
+
self.audio_tower = AutoModel.from_config(config.audio_config)
|
| 535 |
+
language_model_class = (
|
| 536 |
+
Audio8ASRInfiniteQwen2ForCausalLM
|
| 537 |
+
if config.text_config.model_type == Qwen2Config.model_type
|
| 538 |
+
else Audio8ASRInfiniteForCausalLM
|
| 539 |
+
)
|
| 540 |
+
self.language_model = language_model_class(config.text_config)
|
| 541 |
+
self.multi_modal_projector = Audio8ASRInfiniteMaxFrameLenProjector(
|
| 542 |
+
config
|
| 543 |
+
)
|
| 544 |
+
self.time_embedding = VoxtralRealtimeTimeEmbedding(config.text_config.hidden_size)
|
| 545 |
+
self.frame_len_embedding = (
|
| 546 |
+
nn.Embedding(
|
| 547 |
+
len(config.supported_frame_lens),
|
| 548 |
+
config.text_config.hidden_size,
|
| 549 |
+
)
|
| 550 |
+
if config.use_frame_len_embedding
|
| 551 |
+
else None
|
| 552 |
+
)
|
| 553 |
+
self.post_init()
|
| 554 |
+
if self.frame_len_embedding is not None:
|
| 555 |
+
nn.init.normal_(
|
| 556 |
+
self.frame_len_embedding.weight,
|
| 557 |
+
mean=0.0,
|
| 558 |
+
std=float(config.text_config.initializer_range),
|
| 559 |
+
)
|
| 560 |
+
# Semantic VAD heads only exist when the checkpoint declares horizons:
|
| 561 |
+
# a plain transcription checkpoint builds none, so its weight keys are
|
| 562 |
+
# unchanged. One classifier per horizon predicts how many semantic
|
| 563 |
+
# units will appear within that horizon; class 0 is end-of-turn.
|
| 564 |
+
self.semantic_vad_heads: nn.ModuleList | None = None
|
| 565 |
+
self.semantic_vad_horizons_seconds: tuple[float, ...] = ()
|
| 566 |
+
self.semantic_vad_num_classes: int = 0
|
| 567 |
+
configured_horizons = tuple(
|
| 568 |
+
float(horizon)
|
| 569 |
+
for horizon in (
|
| 570 |
+
getattr(config, "semantic_vad_horizons_seconds", None) or ()
|
| 571 |
+
)
|
| 572 |
+
)
|
| 573 |
+
if configured_horizons:
|
| 574 |
+
self.attach_semantic_vad_heads(
|
| 575 |
+
horizons_seconds=configured_horizons,
|
| 576 |
+
num_classes=int(
|
| 577 |
+
getattr(
|
| 578 |
+
config,
|
| 579 |
+
"semantic_vad_num_classes",
|
| 580 |
+
DEFAULT_SEMANTIC_VAD_NUM_CLASSES,
|
| 581 |
+
)
|
| 582 |
+
),
|
| 583 |
+
)
|
| 584 |
+
|
| 585 |
+
def attach_semantic_vad_heads(
|
| 586 |
+
self,
|
| 587 |
+
*,
|
| 588 |
+
horizons_seconds: "Sequence[float]",
|
| 589 |
+
num_classes: int = DEFAULT_SEMANTIC_VAD_NUM_CLASSES,
|
| 590 |
+
) -> nn.ModuleList:
|
| 591 |
+
"""Attach the semantic VAD heads: one "future semantic units" classifier
|
| 592 |
+
per horizon.
|
| 593 |
+
|
| 594 |
+
Each head reads the text backbone's final hidden state and emits that
|
| 595 |
+
horizon's class logits, matching the ``[batch, horizon, token]`` shape of
|
| 596 |
+
the training labels. The horizons and the class count are written back
|
| 597 |
+
into the config so a saved checkpoint rebuilds the same heads on load.
|
| 598 |
+
"""
|
| 599 |
+
|
| 600 |
+
horizons = tuple(float(horizon) for horizon in horizons_seconds)
|
| 601 |
+
if not horizons:
|
| 602 |
+
raise ValueError("semantic VAD horizons must not be empty.")
|
| 603 |
+
num_classes = int(num_classes)
|
| 604 |
+
if num_classes < 2:
|
| 605 |
+
raise ValueError("semantic_vad_num_classes must be at least 2.")
|
| 606 |
+
hidden_size = int(self.config.text_config.hidden_size)
|
| 607 |
+
reference = next(self.language_model.parameters())
|
| 608 |
+
heads = nn.ModuleList(
|
| 609 |
+
[
|
| 610 |
+
nn.Linear(
|
| 611 |
+
hidden_size,
|
| 612 |
+
num_classes,
|
| 613 |
+
bias=True,
|
| 614 |
+
dtype=reference.dtype,
|
| 615 |
+
)
|
| 616 |
+
for _ in horizons
|
| 617 |
+
]
|
| 618 |
+
)
|
| 619 |
+
self.semantic_vad_heads = heads
|
| 620 |
+
self.semantic_vad_horizons_seconds = horizons
|
| 621 |
+
self.semantic_vad_num_classes = num_classes
|
| 622 |
+
self.config.semantic_vad_horizons_seconds = list(horizons)
|
| 623 |
+
self.config.semantic_vad_num_classes = num_classes
|
| 624 |
+
return heads
|
| 625 |
+
|
| 626 |
+
def _semantic_vad_hidden_norm(self) -> nn.Module:
|
| 627 |
+
"""Return the text backbone's final-hidden-state norm layer."""
|
| 628 |
+
|
| 629 |
+
text_model = getattr(self.language_model, "model", None)
|
| 630 |
+
norm = getattr(text_model, "norm", None)
|
| 631 |
+
if norm is None:
|
| 632 |
+
raise RuntimeError(
|
| 633 |
+
"Audio8 ASR Infinite text backbone does not expose `norm`; "
|
| 634 |
+
"semantic VAD heads cannot read the final hidden state."
|
| 635 |
+
)
|
| 636 |
+
return norm
|
| 637 |
+
|
| 638 |
+
def get_input_embeddings(self) -> nn.Module:
|
| 639 |
+
return self.language_model.get_input_embeddings()
|
| 640 |
+
|
| 641 |
+
def set_input_embeddings(self, value: nn.Module) -> None:
|
| 642 |
+
self.language_model.set_input_embeddings(value)
|
| 643 |
+
|
| 644 |
+
def get_output_embeddings(self) -> nn.Module:
|
| 645 |
+
return self.language_model.get_output_embeddings()
|
| 646 |
+
|
| 647 |
+
def set_output_embeddings(self, new_embeddings: nn.Module) -> None:
|
| 648 |
+
self.language_model.set_output_embeddings(new_embeddings)
|
| 649 |
+
|
| 650 |
+
def set_decoder(self, decoder: nn.Module) -> None:
|
| 651 |
+
self.language_model.set_decoder(decoder)
|
| 652 |
+
|
| 653 |
+
def get_decoder(self) -> nn.Module:
|
| 654 |
+
return self.language_model.get_decoder()
|
| 655 |
+
|
| 656 |
+
def _prepare_model_inputs(
|
| 657 |
+
self,
|
| 658 |
+
inputs: torch.Tensor | None = None,
|
| 659 |
+
bos_token_id: torch.Tensor | None = None,
|
| 660 |
+
model_kwargs: dict[str, Any] | None = None,
|
| 661 |
+
) -> tuple[torch.Tensor, str | None, dict[str, Any]]:
|
| 662 |
+
inputs, input_name, model_kwargs = super()._prepare_model_inputs(
|
| 663 |
+
inputs,
|
| 664 |
+
bos_token_id,
|
| 665 |
+
model_kwargs,
|
| 666 |
+
)
|
| 667 |
+
input_features = model_kwargs.get("input_features")
|
| 668 |
+
if isinstance(input_features, GeneratorType):
|
| 669 |
+
input_features_generator = model_kwargs.pop("input_features")
|
| 670 |
+
model_kwargs["input_features_generator"] = (
|
| 671 |
+
input_features_generator
|
| 672 |
+
)
|
| 673 |
+
try:
|
| 674 |
+
model_kwargs["input_features"] = next(
|
| 675 |
+
input_features_generator
|
| 676 |
+
)
|
| 677 |
+
except StopIteration:
|
| 678 |
+
self._stream_exhausted = True
|
| 679 |
+
return inputs, input_name, model_kwargs
|
| 680 |
+
|
| 681 |
+
def _has_unfinished_sequences(
|
| 682 |
+
self,
|
| 683 |
+
this_peer_finished: bool,
|
| 684 |
+
synced_gpus: bool,
|
| 685 |
+
device: torch.device,
|
| 686 |
+
) -> bool:
|
| 687 |
+
if getattr(self, "_stream_exhausted", False):
|
| 688 |
+
self._stream_exhausted = False
|
| 689 |
+
return False
|
| 690 |
+
return super()._has_unfinished_sequences(
|
| 691 |
+
this_peer_finished,
|
| 692 |
+
synced_gpus,
|
| 693 |
+
device,
|
| 694 |
+
)
|
| 695 |
+
|
| 696 |
+
def _update_model_kwargs_for_generation(
|
| 697 |
+
self,
|
| 698 |
+
outputs: Any,
|
| 699 |
+
model_kwargs: dict[str, Any],
|
| 700 |
+
is_encoder_decoder: bool = False,
|
| 701 |
+
num_new_tokens: int = 1,
|
| 702 |
+
) -> dict[str, Any]:
|
| 703 |
+
model_kwargs = super()._update_model_kwargs_for_generation(
|
| 704 |
+
outputs,
|
| 705 |
+
model_kwargs,
|
| 706 |
+
is_encoder_decoder,
|
| 707 |
+
num_new_tokens,
|
| 708 |
+
)
|
| 709 |
+
if hasattr(outputs, "encoder_past_key_values"):
|
| 710 |
+
model_kwargs["encoder_past_key_values"] = (
|
| 711 |
+
outputs.encoder_past_key_values
|
| 712 |
+
)
|
| 713 |
+
if hasattr(outputs, "padding_cache"):
|
| 714 |
+
model_kwargs["padding_cache"] = outputs.padding_cache
|
| 715 |
+
|
| 716 |
+
input_features_generator = model_kwargs.get(
|
| 717 |
+
"input_features_generator"
|
| 718 |
+
)
|
| 719 |
+
if input_features_generator is not None:
|
| 720 |
+
try:
|
| 721 |
+
model_kwargs["input_features"] = next(
|
| 722 |
+
input_features_generator
|
| 723 |
+
)
|
| 724 |
+
except StopIteration:
|
| 725 |
+
self._stream_exhausted = True
|
| 726 |
+
return model_kwargs
|
| 727 |
+
|
| 728 |
+
def _prepare_generation_config(
|
| 729 |
+
self,
|
| 730 |
+
generation_config: Any,
|
| 731 |
+
**kwargs: Any,
|
| 732 |
+
) -> tuple[Any, dict[str, Any]]:
|
| 733 |
+
generation_config, model_kwargs = (
|
| 734 |
+
super()._prepare_generation_config(
|
| 735 |
+
generation_config,
|
| 736 |
+
**kwargs,
|
| 737 |
+
)
|
| 738 |
+
)
|
| 739 |
+
if isinstance(
|
| 740 |
+
model_kwargs.get("input_features"),
|
| 741 |
+
GeneratorType,
|
| 742 |
+
):
|
| 743 |
+
generation_config.max_new_tokens = None
|
| 744 |
+
generation_config.max_length = int(1e9)
|
| 745 |
+
generation_config._voxtral_set_max_length = True
|
| 746 |
+
return generation_config, model_kwargs
|
| 747 |
+
|
| 748 |
+
def _prepare_generated_length(
|
| 749 |
+
self,
|
| 750 |
+
generation_config: Any,
|
| 751 |
+
has_default_max_length: bool,
|
| 752 |
+
has_default_min_length: bool,
|
| 753 |
+
model_input_name: str,
|
| 754 |
+
input_ids_length: int,
|
| 755 |
+
inputs_tensor: torch.Tensor,
|
| 756 |
+
) -> Any:
|
| 757 |
+
if getattr(
|
| 758 |
+
generation_config,
|
| 759 |
+
"_voxtral_set_max_length",
|
| 760 |
+
False,
|
| 761 |
+
):
|
| 762 |
+
has_default_max_length = False
|
| 763 |
+
return super()._prepare_generated_length(
|
| 764 |
+
generation_config,
|
| 765 |
+
has_default_max_length,
|
| 766 |
+
has_default_min_length,
|
| 767 |
+
model_input_name,
|
| 768 |
+
input_ids_length,
|
| 769 |
+
inputs_tensor,
|
| 770 |
+
)
|
| 771 |
+
|
| 772 |
+
def resolve_frame_lens(
|
| 773 |
+
self,
|
| 774 |
+
frame_len: int | torch.Tensor | None,
|
| 775 |
+
*,
|
| 776 |
+
batch_size: int,
|
| 777 |
+
device: torch.device,
|
| 778 |
+
) -> torch.LongTensor:
|
| 779 |
+
# frame_lens has one value per model row. Several rows may refer to the
|
| 780 |
+
# same source audio while using different frame lengths (and delays).
|
| 781 |
+
if frame_len is None:
|
| 782 |
+
frame_len = self.config.supported_frame_lens[0]
|
| 783 |
+
if torch.is_tensor(frame_len):
|
| 784 |
+
frame_lens = frame_len.to(
|
| 785 |
+
device=device,
|
| 786 |
+
dtype=torch.long,
|
| 787 |
+
).view(-1)
|
| 788 |
+
if frame_lens.numel() == 1:
|
| 789 |
+
frame_lens = frame_lens.expand(batch_size)
|
| 790 |
+
elif frame_lens.numel() != batch_size:
|
| 791 |
+
raise ValueError(
|
| 792 |
+
"frame_len tensor must contain 1 or batch_size values, "
|
| 793 |
+
f"got {frame_lens.numel()}."
|
| 794 |
+
)
|
| 795 |
+
else:
|
| 796 |
+
frame_lens = torch.full(
|
| 797 |
+
(batch_size,),
|
| 798 |
+
int(frame_len),
|
| 799 |
+
device=device,
|
| 800 |
+
dtype=torch.long,
|
| 801 |
+
)
|
| 802 |
+
supported = torch.tensor(
|
| 803 |
+
self.config.supported_frame_lens,
|
| 804 |
+
device=device,
|
| 805 |
+
dtype=torch.long,
|
| 806 |
+
)
|
| 807 |
+
if not torch.isin(frame_lens, supported).all():
|
| 808 |
+
raise ValueError(
|
| 809 |
+
"frame_len values must be drawn from "
|
| 810 |
+
f"{self.config.supported_frame_lens}."
|
| 811 |
+
)
|
| 812 |
+
return frame_lens
|
| 813 |
+
|
| 814 |
+
def group_audio_hidden_states(
|
| 815 |
+
self,
|
| 816 |
+
audio_hidden_states: torch.Tensor,
|
| 817 |
+
*,
|
| 818 |
+
frame_len: int | torch.Tensor | None,
|
| 819 |
+
target_token_count: int | None = None,
|
| 820 |
+
) -> torch.Tensor:
|
| 821 |
+
frame_lens = self.resolve_frame_lens(
|
| 822 |
+
frame_len,
|
| 823 |
+
batch_size=audio_hidden_states.shape[0],
|
| 824 |
+
device=audio_hidden_states.device,
|
| 825 |
+
)
|
| 826 |
+
max_frame_len = int(self.config.max_frame_len)
|
| 827 |
+
hidden_size = int(self.config.audio_config.hidden_size)
|
| 828 |
+
grouped_batches: list[tuple[torch.Tensor, torch.Tensor]] = []
|
| 829 |
+
# Partition the expanded batch by frame length. A row is processed by
|
| 830 |
+
# exactly one branch; supported frame lengths are not fused together.
|
| 831 |
+
for row_frame_len in self.config.supported_frame_lens:
|
| 832 |
+
row_indices = torch.nonzero(
|
| 833 |
+
frame_lens == row_frame_len,
|
| 834 |
+
as_tuple=False,
|
| 835 |
+
).flatten()
|
| 836 |
+
if row_indices.numel() == 0:
|
| 837 |
+
continue
|
| 838 |
+
rows = audio_hidden_states.index_select(0, row_indices)
|
| 839 |
+
# Complete the last temporal group before reshaping consecutive
|
| 840 |
+
# audio-tower frames into one streaming-token group.
|
| 841 |
+
temporal_padding = (-rows.shape[1]) % row_frame_len
|
| 842 |
+
if temporal_padding:
|
| 843 |
+
rows = F.pad(rows, (0, 0, 0, temporal_padding))
|
| 844 |
+
grouped = rows.reshape(
|
| 845 |
+
rows.shape[0],
|
| 846 |
+
-1,
|
| 847 |
+
row_frame_len,
|
| 848 |
+
hidden_size,
|
| 849 |
+
)
|
| 850 |
+
# Every gear shares one projector. Pad the frame slots inside each
|
| 851 |
+
# group so its flattened width is always max_frame_len * hidden_size.
|
| 852 |
+
if row_frame_len < max_frame_len:
|
| 853 |
+
grouped = F.pad(
|
| 854 |
+
grouped,
|
| 855 |
+
(0, 0, 0, max_frame_len - row_frame_len),
|
| 856 |
+
)
|
| 857 |
+
grouped_batches.append(
|
| 858 |
+
(
|
| 859 |
+
row_indices,
|
| 860 |
+
grouped.reshape(
|
| 861 |
+
rows.shape[0],
|
| 862 |
+
-1,
|
| 863 |
+
max_frame_len * hidden_size,
|
| 864 |
+
),
|
| 865 |
+
)
|
| 866 |
+
)
|
| 867 |
+
|
| 868 |
+
# Text rows share one padded sequence length. Each gear therefore pads
|
| 869 |
+
# or truncates its number of grouped audio tokens to that same length.
|
| 870 |
+
max_token_count = (
|
| 871 |
+
int(target_token_count)
|
| 872 |
+
if target_token_count is not None
|
| 873 |
+
else max(grouped.shape[1] for _, grouped in grouped_batches)
|
| 874 |
+
)
|
| 875 |
+
projector_inputs = audio_hidden_states.new_zeros(
|
| 876 |
+
audio_hidden_states.shape[0],
|
| 877 |
+
max_token_count,
|
| 878 |
+
int(self.config.projection_size),
|
| 879 |
+
)
|
| 880 |
+
for row_indices, grouped in grouped_batches:
|
| 881 |
+
if grouped.shape[1] < max_token_count:
|
| 882 |
+
grouped = F.pad(
|
| 883 |
+
grouped,
|
| 884 |
+
(0, 0, 0, max_token_count - grouped.shape[1]),
|
| 885 |
+
)
|
| 886 |
+
else:
|
| 887 |
+
grouped = grouped[:, :max_token_count]
|
| 888 |
+
# Restore the original expanded-batch order after per-gear work.
|
| 889 |
+
projector_inputs.index_copy_(0, row_indices, grouped)
|
| 890 |
+
return projector_inputs
|
| 891 |
+
|
| 892 |
+
def get_audio_tower_hidden_states(
|
| 893 |
+
self,
|
| 894 |
+
input_features: torch.FloatTensor | None = None,
|
| 895 |
+
padding_cache: Any | None = None,
|
| 896 |
+
encoder_inputs_embeds: torch.FloatTensor | None = None,
|
| 897 |
+
past_key_values: Any | None = None,
|
| 898 |
+
use_cache: bool | None = None,
|
| 899 |
+
return_outputs: bool = False,
|
| 900 |
+
**kwargs: Any,
|
| 901 |
+
) -> torch.Tensor | tuple[torch.Tensor, Any]:
|
| 902 |
+
"""Run the frozen-capable audio tower at its native frame clock."""
|
| 903 |
+
|
| 904 |
+
if (input_features is None) == (encoder_inputs_embeds is None):
|
| 905 |
+
raise ValueError("Specify exactly one of input_features or encoder_inputs_embeds.")
|
| 906 |
+
|
| 907 |
+
audio_outputs = self.audio_tower(
|
| 908 |
+
input_features=input_features,
|
| 909 |
+
inputs_embeds=encoder_inputs_embeds,
|
| 910 |
+
past_key_values=past_key_values,
|
| 911 |
+
padding_cache=padding_cache,
|
| 912 |
+
return_dict=True,
|
| 913 |
+
use_cache=use_cache,
|
| 914 |
+
use_padding_cache=use_cache,
|
| 915 |
+
**kwargs,
|
| 916 |
+
)
|
| 917 |
+
if return_outputs:
|
| 918 |
+
return audio_outputs.last_hidden_state, audio_outputs
|
| 919 |
+
return audio_outputs.last_hidden_state
|
| 920 |
+
|
| 921 |
+
def get_audio_projector_input_features(
|
| 922 |
+
self,
|
| 923 |
+
input_features: torch.FloatTensor | None = None,
|
| 924 |
+
padding_cache: Any | None = None,
|
| 925 |
+
encoder_inputs_embeds: torch.FloatTensor | None = None,
|
| 926 |
+
past_key_values: Any | None = None,
|
| 927 |
+
use_cache: bool | None = None,
|
| 928 |
+
frame_len: int | torch.Tensor | None = None,
|
| 929 |
+
target_token_count: int | None = None,
|
| 930 |
+
return_outputs: bool = False,
|
| 931 |
+
**kwargs: Any,
|
| 932 |
+
) -> torch.Tensor | tuple[torch.Tensor, Any]:
|
| 933 |
+
"""Run the audio tower and group states for the max-frame-len projector."""
|
| 934 |
+
|
| 935 |
+
audio_hidden_states, audio_outputs = (
|
| 936 |
+
self.get_audio_tower_hidden_states(
|
| 937 |
+
input_features=input_features,
|
| 938 |
+
encoder_inputs_embeds=encoder_inputs_embeds,
|
| 939 |
+
past_key_values=past_key_values,
|
| 940 |
+
padding_cache=padding_cache,
|
| 941 |
+
use_cache=use_cache,
|
| 942 |
+
return_outputs=True,
|
| 943 |
+
**kwargs,
|
| 944 |
+
)
|
| 945 |
+
)
|
| 946 |
+
projector_inputs = self.group_audio_hidden_states(
|
| 947 |
+
audio_hidden_states,
|
| 948 |
+
frame_len=frame_len,
|
| 949 |
+
target_token_count=target_token_count,
|
| 950 |
+
)
|
| 951 |
+
if return_outputs:
|
| 952 |
+
return projector_inputs, audio_outputs
|
| 953 |
+
return projector_inputs
|
| 954 |
+
|
| 955 |
+
def get_audio_features(
|
| 956 |
+
self,
|
| 957 |
+
input_features: torch.FloatTensor | None = None,
|
| 958 |
+
padding_cache: Any | None = None,
|
| 959 |
+
encoder_inputs_embeds: torch.FloatTensor | None = None,
|
| 960 |
+
past_key_values: Any | None = None,
|
| 961 |
+
use_cache: bool | None = None,
|
| 962 |
+
frame_len: int | torch.Tensor | None = None,
|
| 963 |
+
target_token_count: int | None = None,
|
| 964 |
+
return_outputs: bool = False,
|
| 965 |
+
**kwargs: Any,
|
| 966 |
+
) -> torch.Tensor | tuple[torch.Tensor, Any]:
|
| 967 |
+
audio_hidden_states, audio_outputs = (
|
| 968 |
+
self.get_audio_projector_input_features(
|
| 969 |
+
input_features=input_features,
|
| 970 |
+
encoder_inputs_embeds=encoder_inputs_embeds,
|
| 971 |
+
past_key_values=past_key_values,
|
| 972 |
+
padding_cache=padding_cache,
|
| 973 |
+
use_cache=use_cache,
|
| 974 |
+
frame_len=frame_len,
|
| 975 |
+
target_token_count=target_token_count,
|
| 976 |
+
return_outputs=True,
|
| 977 |
+
**kwargs,
|
| 978 |
+
)
|
| 979 |
+
)
|
| 980 |
+
audio_embeds = self.multi_modal_projector(
|
| 981 |
+
audio_hidden_states
|
| 982 |
+
)
|
| 983 |
+
audio_outputs.pooler_output = audio_embeds
|
| 984 |
+
if return_outputs:
|
| 985 |
+
return audio_embeds, audio_outputs
|
| 986 |
+
return audio_embeds
|
| 987 |
+
|
| 988 |
+
def get_source_audio_embeds(
|
| 989 |
+
self,
|
| 990 |
+
source_input_features: torch.FloatTensor,
|
| 991 |
+
audio_source_indices: torch.LongTensor,
|
| 992 |
+
*,
|
| 993 |
+
frame_len: int | torch.Tensor | None = None,
|
| 994 |
+
target_token_count: int | None = None,
|
| 995 |
+
) -> torch.Tensor:
|
| 996 |
+
"""Compute the audio tower result once and fan it out to expanded rows.
|
| 997 |
+
|
| 998 |
+
``forward(source_input_features=...)`` already computes the audio tower
|
| 999 |
+
once per call and then fans its states out to expanded rows. This method
|
| 1000 |
+
exposes that operation explicitly so callers that need several
|
| 1001 |
+
language-model forwards can share one audio result.
|
| 1002 |
+
"""
|
| 1003 |
+
audio_hidden_states = self.get_audio_tower_hidden_states(
|
| 1004 |
+
input_features=source_input_features,
|
| 1005 |
+
use_cache=False,
|
| 1006 |
+
)
|
| 1007 |
+
expanded_audio_hidden_states = audio_hidden_states.index_select(
|
| 1008 |
+
0,
|
| 1009 |
+
audio_source_indices.to(device=audio_hidden_states.device),
|
| 1010 |
+
)
|
| 1011 |
+
projector_inputs = self.group_audio_hidden_states(
|
| 1012 |
+
expanded_audio_hidden_states,
|
| 1013 |
+
frame_len=frame_len,
|
| 1014 |
+
target_token_count=target_token_count,
|
| 1015 |
+
)
|
| 1016 |
+
return self.multi_modal_projector(projector_inputs)
|
| 1017 |
+
|
| 1018 |
+
def build_text_inputs_embeds(
|
| 1019 |
+
self,
|
| 1020 |
+
*,
|
| 1021 |
+
input_ids: torch.LongTensor | None = None,
|
| 1022 |
+
inputs_embeds: torch.FloatTensor | None = None,
|
| 1023 |
+
audio_embeds: torch.FloatTensor | None = None,
|
| 1024 |
+
) -> torch.FloatTensor:
|
| 1025 |
+
if (input_ids is None) ^ (inputs_embeds is not None):
|
| 1026 |
+
raise ValueError("Specify exactly one of input_ids or inputs_embeds.")
|
| 1027 |
+
if inputs_embeds is None:
|
| 1028 |
+
inputs_embeds = self.get_input_embeddings()(input_ids)
|
| 1029 |
+
if audio_embeds is not None:
|
| 1030 |
+
audio_embeds = audio_embeds.to(device=inputs_embeds.device, dtype=inputs_embeds.dtype)
|
| 1031 |
+
if audio_embeds.shape[:2] != inputs_embeds.shape[:2]:
|
| 1032 |
+
raise ValueError(
|
| 1033 |
+
"Audio embedding shape must match token embedding shape before fusion: "
|
| 1034 |
+
f"audio={tuple(audio_embeds.shape)} tokens={tuple(inputs_embeds.shape)}"
|
| 1035 |
+
)
|
| 1036 |
+
inputs_embeds = inputs_embeds + audio_embeds
|
| 1037 |
+
return inputs_embeds
|
| 1038 |
+
|
| 1039 |
+
def build_t_cond(
|
| 1040 |
+
self,
|
| 1041 |
+
num_delay_tokens: int | torch.Tensor | None,
|
| 1042 |
+
*,
|
| 1043 |
+
batch_size: int,
|
| 1044 |
+
device: torch.device,
|
| 1045 |
+
dtype: torch.dtype,
|
| 1046 |
+
frame_len: int | torch.Tensor | None = None,
|
| 1047 |
+
) -> torch.Tensor:
|
| 1048 |
+
if num_delay_tokens is None:
|
| 1049 |
+
num_delay_tokens = self.config.default_num_delay_tokens
|
| 1050 |
+
if num_delay_tokens is None:
|
| 1051 |
+
raise ValueError("Audio8 ASR Infinite requires explicit `num_delay_tokens`.")
|
| 1052 |
+
if torch.is_tensor(num_delay_tokens):
|
| 1053 |
+
delay_values = num_delay_tokens.to(device=device, dtype=dtype).view(-1)
|
| 1054 |
+
if delay_values.numel() == 1:
|
| 1055 |
+
delay_values = delay_values.expand(batch_size)
|
| 1056 |
+
elif delay_values.numel() != batch_size:
|
| 1057 |
+
raise ValueError(
|
| 1058 |
+
f"num_delay_tokens tensor must contain 1 or batch_size values, got {delay_values.numel()}."
|
| 1059 |
+
)
|
| 1060 |
+
else:
|
| 1061 |
+
delay_values = torch.full((batch_size,), float(num_delay_tokens), device=device, dtype=dtype)
|
| 1062 |
+
# The time embedding is sinusoidal. Evaluating it row by row creates
|
| 1063 |
+
# three tiny GPU kernels (and Python iteration) for every LM window;
|
| 1064 |
+
# build the same [batch, hidden] tensor in one vectorized operation.
|
| 1065 |
+
inv_freq = self.time_embedding.inv_freq.to(
|
| 1066 |
+
device=device,
|
| 1067 |
+
dtype=dtype,
|
| 1068 |
+
)
|
| 1069 |
+
phase = delay_values.unsqueeze(-1) * inv_freq.unsqueeze(0)
|
| 1070 |
+
delay_embeddings = torch.cat(
|
| 1071 |
+
(phase.cos(), phase.sin()),
|
| 1072 |
+
dim=-1,
|
| 1073 |
+
)
|
| 1074 |
+
if self.frame_len_embedding is not None:
|
| 1075 |
+
frame_lens = self.resolve_frame_lens(
|
| 1076 |
+
frame_len,
|
| 1077 |
+
batch_size=batch_size,
|
| 1078 |
+
device=device,
|
| 1079 |
+
)
|
| 1080 |
+
frame_len_indices = torch.empty_like(frame_lens)
|
| 1081 |
+
for index, supported_frame_len in enumerate(
|
| 1082 |
+
self.config.supported_frame_lens
|
| 1083 |
+
):
|
| 1084 |
+
frame_len_indices[
|
| 1085 |
+
frame_lens == supported_frame_len
|
| 1086 |
+
] = index
|
| 1087 |
+
delay_embeddings = delay_embeddings + self.frame_len_embedding(
|
| 1088 |
+
frame_len_indices
|
| 1089 |
+
).to(dtype=delay_embeddings.dtype)
|
| 1090 |
+
return delay_embeddings.unsqueeze(1)
|
| 1091 |
+
|
| 1092 |
+
def forward_language_model_with_delay(
|
| 1093 |
+
self,
|
| 1094 |
+
*,
|
| 1095 |
+
inputs_embeds: torch.FloatTensor,
|
| 1096 |
+
attention_mask: torch.Tensor | None = None,
|
| 1097 |
+
position_ids: torch.LongTensor | None = None,
|
| 1098 |
+
past_key_values: Any | None = None,
|
| 1099 |
+
use_cache: bool | None = None,
|
| 1100 |
+
logits_to_keep: int | torch.Tensor = 0,
|
| 1101 |
+
num_delay_tokens: int | torch.Tensor | None = None,
|
| 1102 |
+
frame_len: int | torch.Tensor | None = None,
|
| 1103 |
+
t_cond: torch.Tensor | None = None,
|
| 1104 |
+
**kwargs: Any,
|
| 1105 |
+
) -> CausalLMOutputWithPast:
|
| 1106 |
+
if t_cond is None:
|
| 1107 |
+
t_cond = self.build_t_cond(
|
| 1108 |
+
num_delay_tokens,
|
| 1109 |
+
batch_size=inputs_embeds.shape[0],
|
| 1110 |
+
device=inputs_embeds.device,
|
| 1111 |
+
dtype=inputs_embeds.dtype,
|
| 1112 |
+
frame_len=frame_len,
|
| 1113 |
+
)
|
| 1114 |
+
return self.language_model(
|
| 1115 |
+
attention_mask=attention_mask,
|
| 1116 |
+
position_ids=position_ids,
|
| 1117 |
+
past_key_values=past_key_values,
|
| 1118 |
+
inputs_embeds=inputs_embeds,
|
| 1119 |
+
use_cache=use_cache,
|
| 1120 |
+
logits_to_keep=logits_to_keep,
|
| 1121 |
+
t_cond=t_cond,
|
| 1122 |
+
**kwargs,
|
| 1123 |
+
)
|
| 1124 |
+
|
| 1125 |
+
def forward(
|
| 1126 |
+
self,
|
| 1127 |
+
input_ids: torch.LongTensor | None = None,
|
| 1128 |
+
input_features: torch.FloatTensor | None = None,
|
| 1129 |
+
source_input_features: torch.FloatTensor | None = None,
|
| 1130 |
+
audio_source_indices: torch.LongTensor | None = None,
|
| 1131 |
+
attention_mask: torch.Tensor | None = None,
|
| 1132 |
+
position_ids: torch.LongTensor | None = None,
|
| 1133 |
+
past_key_values: Any | None = None,
|
| 1134 |
+
encoder_past_key_values: Any | None = None,
|
| 1135 |
+
padding_cache: Any | None = None,
|
| 1136 |
+
inputs_embeds: torch.FloatTensor | None = None,
|
| 1137 |
+
encoder_inputs_embeds: torch.FloatTensor | None = None,
|
| 1138 |
+
use_cache: bool | None = None,
|
| 1139 |
+
logits_to_keep: int | torch.Tensor = 0,
|
| 1140 |
+
num_delay_tokens: int | torch.Tensor | None = None,
|
| 1141 |
+
frame_len: int | torch.Tensor | None = None,
|
| 1142 |
+
**kwargs: Any,
|
| 1143 |
+
) -> CausalLMOutputWithPast:
|
| 1144 |
+
audio_input_count = sum(
|
| 1145 |
+
value is not None
|
| 1146 |
+
for value in (
|
| 1147 |
+
input_features,
|
| 1148 |
+
source_input_features,
|
| 1149 |
+
encoder_inputs_embeds,
|
| 1150 |
+
)
|
| 1151 |
+
)
|
| 1152 |
+
if audio_input_count != 1:
|
| 1153 |
+
raise ValueError(
|
| 1154 |
+
"Specify exactly one of input_features, source_input_features, "
|
| 1155 |
+
"or encoder_inputs_embeds."
|
| 1156 |
+
)
|
| 1157 |
+
uses_source_rows = source_input_features is not None
|
| 1158 |
+
if not uses_source_rows:
|
| 1159 |
+
if audio_source_indices is not None:
|
| 1160 |
+
raise ValueError(
|
| 1161 |
+
"audio_source_indices is only valid with source audio rows."
|
| 1162 |
+
)
|
| 1163 |
+
audio_features = input_features
|
| 1164 |
+
else:
|
| 1165 |
+
if audio_source_indices is None:
|
| 1166 |
+
raise ValueError(
|
| 1167 |
+
"Source audio rows require audio_source_indices."
|
| 1168 |
+
)
|
| 1169 |
+
if not torch.is_tensor(audio_source_indices):
|
| 1170 |
+
raise TypeError("audio_source_indices must be a tensor.")
|
| 1171 |
+
if audio_source_indices.dtype != torch.long:
|
| 1172 |
+
raise TypeError("audio_source_indices must have dtype torch.long.")
|
| 1173 |
+
if audio_source_indices.ndim != 1:
|
| 1174 |
+
raise ValueError("audio_source_indices must be one-dimensional.")
|
| 1175 |
+
expanded_batch_size = (
|
| 1176 |
+
input_ids.shape[0]
|
| 1177 |
+
if input_ids is not None
|
| 1178 |
+
else inputs_embeds.shape[0]
|
| 1179 |
+
if inputs_embeds is not None
|
| 1180 |
+
else None
|
| 1181 |
+
)
|
| 1182 |
+
if expanded_batch_size is None:
|
| 1183 |
+
raise ValueError(
|
| 1184 |
+
"source_input_features requires input_ids or inputs_embeds."
|
| 1185 |
+
)
|
| 1186 |
+
if audio_source_indices.numel() != expanded_batch_size:
|
| 1187 |
+
raise ValueError(
|
| 1188 |
+
"audio_source_indices length must match the expanded text batch: "
|
| 1189 |
+
f"indices={audio_source_indices.numel()} batch={expanded_batch_size}."
|
| 1190 |
+
)
|
| 1191 |
+
source_batch_size = source_input_features.shape[0]
|
| 1192 |
+
if source_batch_size <= 0:
|
| 1193 |
+
raise ValueError("Source audio rows must contain at least one row.")
|
| 1194 |
+
if (
|
| 1195 |
+
audio_source_indices.device.type == "cpu"
|
| 1196 |
+
and audio_source_indices.numel()
|
| 1197 |
+
and (
|
| 1198 |
+
int(audio_source_indices.min().item()) < 0
|
| 1199 |
+
or int(audio_source_indices.max().item()) >= source_batch_size
|
| 1200 |
+
)
|
| 1201 |
+
):
|
| 1202 |
+
raise ValueError(
|
| 1203 |
+
"audio_source_indices contains an out-of-range source row."
|
| 1204 |
+
)
|
| 1205 |
+
audio_features = source_input_features
|
| 1206 |
+
|
| 1207 |
+
if source_input_features is not None:
|
| 1208 |
+
# The collator stores each source waveform once, then expands its
|
| 1209 |
+
# text targets across frame-length/delay configurations. Run the
|
| 1210 |
+
# audio tower once per source and fan its states out to those rows.
|
| 1211 |
+
audio_hidden_states, audio_outputs = (
|
| 1212 |
+
self.get_audio_tower_hidden_states(
|
| 1213 |
+
input_features=source_input_features,
|
| 1214 |
+
past_key_values=encoder_past_key_values,
|
| 1215 |
+
padding_cache=padding_cache,
|
| 1216 |
+
use_cache=use_cache,
|
| 1217 |
+
return_outputs=True,
|
| 1218 |
+
)
|
| 1219 |
+
)
|
| 1220 |
+
expanded_audio_hidden_states = (
|
| 1221 |
+
audio_hidden_states.index_select(
|
| 1222 |
+
0,
|
| 1223 |
+
audio_source_indices.to(
|
| 1224 |
+
device=audio_hidden_states.device
|
| 1225 |
+
),
|
| 1226 |
+
)
|
| 1227 |
+
)
|
| 1228 |
+
projector_inputs = self.group_audio_hidden_states(
|
| 1229 |
+
expanded_audio_hidden_states,
|
| 1230 |
+
frame_len=frame_len,
|
| 1231 |
+
target_token_count=(
|
| 1232 |
+
input_ids.shape[1]
|
| 1233 |
+
if input_ids is not None
|
| 1234 |
+
else inputs_embeds.shape[1]
|
| 1235 |
+
),
|
| 1236 |
+
)
|
| 1237 |
+
audio_embeds = self.multi_modal_projector(
|
| 1238 |
+
projector_inputs
|
| 1239 |
+
)
|
| 1240 |
+
else:
|
| 1241 |
+
audio_embeds, audio_outputs = self.get_audio_features(
|
| 1242 |
+
input_features=audio_features,
|
| 1243 |
+
encoder_inputs_embeds=encoder_inputs_embeds,
|
| 1244 |
+
past_key_values=encoder_past_key_values,
|
| 1245 |
+
padding_cache=padding_cache,
|
| 1246 |
+
use_cache=use_cache,
|
| 1247 |
+
frame_len=frame_len,
|
| 1248 |
+
target_token_count=(
|
| 1249 |
+
input_ids.shape[1]
|
| 1250 |
+
if input_ids is not None
|
| 1251 |
+
else inputs_embeds.shape[1]
|
| 1252 |
+
),
|
| 1253 |
+
return_outputs=True,
|
| 1254 |
+
)
|
| 1255 |
+
inputs_embeds = self.build_text_inputs_embeds(
|
| 1256 |
+
input_ids=input_ids,
|
| 1257 |
+
inputs_embeds=inputs_embeds,
|
| 1258 |
+
audio_embeds=audio_embeds,
|
| 1259 |
+
)
|
| 1260 |
+
# Semantic VAD heads read the text backbone norm layer's output (the
|
| 1261 |
+
# final hidden state). A forward hook is used instead of
|
| 1262 |
+
# output_hidden_states: only the last layer is needed, so no other
|
| 1263 |
+
# activations are retained.
|
| 1264 |
+
captured_final_hidden_state: dict[str, torch.Tensor] = {}
|
| 1265 |
+
semantic_vad_hook: Any | None = None
|
| 1266 |
+
if self.semantic_vad_heads is not None:
|
| 1267 |
+
|
| 1268 |
+
def _capture_final_hidden_state(
|
| 1269 |
+
_module: nn.Module,
|
| 1270 |
+
_hook_inputs: tuple[Any, ...],
|
| 1271 |
+
output: torch.Tensor,
|
| 1272 |
+
) -> None:
|
| 1273 |
+
captured_final_hidden_state["final"] = output
|
| 1274 |
+
|
| 1275 |
+
semantic_vad_hook = (
|
| 1276 |
+
self._semantic_vad_hidden_norm().register_forward_hook(
|
| 1277 |
+
_capture_final_hidden_state
|
| 1278 |
+
)
|
| 1279 |
+
)
|
| 1280 |
+
try:
|
| 1281 |
+
outputs = self.forward_language_model_with_delay(
|
| 1282 |
+
inputs_embeds=inputs_embeds,
|
| 1283 |
+
attention_mask=attention_mask,
|
| 1284 |
+
position_ids=position_ids,
|
| 1285 |
+
past_key_values=past_key_values,
|
| 1286 |
+
use_cache=use_cache,
|
| 1287 |
+
logits_to_keep=logits_to_keep,
|
| 1288 |
+
num_delay_tokens=num_delay_tokens,
|
| 1289 |
+
frame_len=frame_len,
|
| 1290 |
+
**kwargs,
|
| 1291 |
+
)
|
| 1292 |
+
finally:
|
| 1293 |
+
if semantic_vad_hook is not None:
|
| 1294 |
+
semantic_vad_hook.remove()
|
| 1295 |
+
if self.semantic_vad_heads is not None:
|
| 1296 |
+
final_hidden_state = captured_final_hidden_state.get("final")
|
| 1297 |
+
if final_hidden_state is None:
|
| 1298 |
+
raise RuntimeError(
|
| 1299 |
+
"Audio8 ASR Infinite did not expose the final hidden state "
|
| 1300 |
+
"for the semantic VAD heads."
|
| 1301 |
+
)
|
| 1302 |
+
outputs["semantic_vad_logits"] = torch.stack(
|
| 1303 |
+
[head(final_hidden_state) for head in self.semantic_vad_heads],
|
| 1304 |
+
dim=1,
|
| 1305 |
+
)
|
| 1306 |
+
outputs.encoder_past_key_values = (
|
| 1307 |
+
getattr(audio_outputs, "past_key_values", None)
|
| 1308 |
+
if use_cache and audio_outputs is not None
|
| 1309 |
+
else None
|
| 1310 |
+
)
|
| 1311 |
+
outputs.padding_cache = (
|
| 1312 |
+
getattr(audio_outputs, "padding_cache", None)
|
| 1313 |
+
if use_cache and audio_outputs is not None
|
| 1314 |
+
else None
|
| 1315 |
+
)
|
| 1316 |
+
return outputs
|
| 1317 |
+
|
| 1318 |
+
|
| 1319 |
+
__all__ = [
|
| 1320 |
+
"AUDIO8_ASR_INFINITE_WEIGHT_FORMAT_VERSION",
|
| 1321 |
+
"Audio8ASRInfiniteConfig",
|
| 1322 |
+
"Audio8ASRInfiniteForCausalLM",
|
| 1323 |
+
"Audio8ASRInfiniteForConditionalGeneration",
|
| 1324 |
+
"Audio8ASRInfiniteMaxFrameLenProjector",
|
| 1325 |
+
"Audio8ASRInfiniteQwen2ForCausalLM",
|
| 1326 |
+
"Audio8ASRInfiniteQwen2TextModel",
|
| 1327 |
+
"Audio8ASRInfiniteTextModel",
|
| 1328 |
+
"LANGUAGE_EN_TOKEN",
|
| 1329 |
+
"LANGUAGE_ZH_TOKEN",
|
| 1330 |
+
"QWEN_ASR_TEXT_TOKEN",
|
| 1331 |
+
"QWEN_AUDIO_PAD_TOKEN",
|
| 1332 |
+
"Qwen2RealtimeV1DecoderLayer",
|
| 1333 |
+
"STREAMING_PAD_TOKEN",
|
| 1334 |
+
"STREAMING_WORD_TOKEN",
|
| 1335 |
+
"ensure_voxtral_streaming_tokens",
|
| 1336 |
+
"resolve_qwen_language_token_id",
|
| 1337 |
+
"resolve_qwen_streaming_special_token_ids",
|
| 1338 |
+
]
|
preprocessor_config.json
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"feature_extractor_type": "VoxtralRealtimeFeatureExtractor",
|
| 3 |
+
"feature_size": 128,
|
| 4 |
+
"global_log_mel_max": 1.5,
|
| 5 |
+
"hop_length": 160,
|
| 6 |
+
"n_fft": 400,
|
| 7 |
+
"padding_side": "right",
|
| 8 |
+
"padding_value": 0.0,
|
| 9 |
+
"return_attention_mask": true,
|
| 10 |
+
"sampling_rate": 16000,
|
| 11 |
+
"win_length": 400
|
| 12 |
+
}
|
semantic_vad_heads.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d1ff79e0282ef53aae176a42b1c2491815f8d26ef10a1e453dd2e41832adb7f7
|
| 3 |
+
size 131888
|
tokenizer.json
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:72f8b7e878ee6606c309881f68201e277f9dae50175a387074a5181d771b9599
|
| 3 |
+
size 11422657
|
tokenizer_config.json
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"add_prefix_space": false,
|
| 3 |
+
"backend": "tokenizers",
|
| 4 |
+
"bos_token": "<|im_start|>",
|
| 5 |
+
"clean_up_tokenization_spaces": false,
|
| 6 |
+
"eos_token": "<|im_end|>",
|
| 7 |
+
"errors": "replace",
|
| 8 |
+
"extra_special_tokens": [
|
| 9 |
+
"[STREAMING_PAD]",
|
| 10 |
+
"[STREAMING_WORD]",
|
| 11 |
+
"[LANGUAGE_ZH]",
|
| 12 |
+
"[LANGUAGE_EN]"
|
| 13 |
+
],
|
| 14 |
+
"is_local": true,
|
| 15 |
+
"local_files_only": false,
|
| 16 |
+
"model_max_length": 131072,
|
| 17 |
+
"pad_token": "<|endoftext|>",
|
| 18 |
+
"split_special_tokens": false,
|
| 19 |
+
"tokenizer_class": "Qwen2Tokenizer",
|
| 20 |
+
"unk_token": null
|
| 21 |
+
}
|