Upload folder using huggingface_hub
Browse files- .meta.json +2 -2
- README.md +6 -7
- chat_template.jinja +64 -37
- config.json +1 -1
- generation_config.json +1 -1
- tokenizer.json +28 -28
- tokenizer_config.json +4 -3
.meta.json
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
{
|
| 2 |
-
"torch": "2.11.0+
|
| 3 |
-
"transformers": "5.
|
| 4 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"torch": "2.11.0+cu128",
|
| 3 |
+
"transformers": "5.13.0"
|
| 4 |
}
|
README.md
CHANGED
|
@@ -1,10 +1,10 @@
|
|
| 1 |
---
|
| 2 |
library_name: transformers
|
| 3 |
base_model:
|
| 4 |
-
- tencent/Hy3
|
| 5 |
---
|
| 6 |
|
| 7 |
-
This tiny model is intended for debugging. It is randomly initialized using the configuration adapted from [tencent/Hy3
|
| 8 |
|
| 9 |
| File path | Size |
|
| 10 |
|------|------|
|
|
@@ -70,8 +70,7 @@ print(output_text)
|
|
| 70 |
|
| 71 |
### Codes to create this repo:
|
| 72 |
|
| 73 |
-
<details>
|
| 74 |
-
<summary>Click to expand</summary>
|
| 75 |
|
| 76 |
```python
|
| 77 |
import json
|
|
@@ -90,7 +89,7 @@ from transformers import (
|
|
| 90 |
set_seed,
|
| 91 |
)
|
| 92 |
|
| 93 |
-
source_model_id = "tencent/Hy3
|
| 94 |
save_folder = "/tmp/tiny-random/hy3"
|
| 95 |
|
| 96 |
processor = AutoTokenizer.from_pretrained(source_model_id, trust_remote_code=True)
|
|
@@ -246,5 +245,5 @@ HYV3ForCausalLM(
|
|
| 246 |
|
| 247 |
### Test environment:
|
| 248 |
|
| 249 |
-
- torch: 2.11.0+
|
| 250 |
-
- transformers: 5.
|
|
|
|
| 1 |
---
|
| 2 |
library_name: transformers
|
| 3 |
base_model:
|
| 4 |
+
- tencent/Hy3
|
| 5 |
---
|
| 6 |
|
| 7 |
+
This tiny model is intended for debugging. It is randomly initialized using the configuration adapted from [tencent/Hy3](https://huggingface.co/tencent/Hy3).
|
| 8 |
|
| 9 |
| File path | Size |
|
| 10 |
|------|------|
|
|
|
|
| 70 |
|
| 71 |
### Codes to create this repo:
|
| 72 |
|
| 73 |
+
<details><summary>Click to expand</summary>
|
|
|
|
| 74 |
|
| 75 |
```python
|
| 76 |
import json
|
|
|
|
| 89 |
set_seed,
|
| 90 |
)
|
| 91 |
|
| 92 |
+
source_model_id = "tencent/Hy3"
|
| 93 |
save_folder = "/tmp/tiny-random/hy3"
|
| 94 |
|
| 95 |
processor = AutoTokenizer.from_pretrained(source_model_id, trust_remote_code=True)
|
|
|
|
| 245 |
|
| 246 |
### Test environment:
|
| 247 |
|
| 248 |
+
- torch: 2.11.0+cu128
|
| 249 |
+
- transformers: 5.13.0
|
chat_template.jinja
CHANGED
|
@@ -1,40 +1,58 @@
|
|
| 1 |
{#- ----------‑‑‑ special token variables ‑‑‑---------- -#}
|
| 2 |
-
{%- set
|
| 3 |
-
{%- set
|
| 4 |
-
{%- set
|
| 5 |
-
{%- set
|
| 6 |
-
{%- set
|
| 7 |
-
{%- set
|
| 8 |
-
{%- set
|
| 9 |
-
{%- set
|
| 10 |
-
{%- set
|
| 11 |
-
{%- set
|
| 12 |
-
{%- set
|
| 13 |
-
{%- set
|
| 14 |
-
{%- set
|
| 15 |
-
{%- set
|
| 16 |
-
{%- set
|
| 17 |
-
{%- set
|
| 18 |
-
{%- set
|
| 19 |
-
{%- set
|
| 20 |
-
{%- set
|
| 21 |
-
{%- set
|
| 22 |
-
{%- set
|
|
|
|
|
|
|
| 23 |
{#- ----------‑‑‑ hyperparameters variables ‑‑‑---------- -#}
|
| 24 |
{%- if not add_generation_prompt is defined %}
|
| 25 |
{%- set add_generation_prompt = false %}
|
| 26 |
{%- endif %}
|
| 27 |
-
{%- if not
|
| 28 |
-
{%-
|
| 29 |
-
{%-
|
| 30 |
-
{%-
|
| 31 |
-
|
|
|
|
| 32 |
{%- endif %}
|
| 33 |
{%- if not is_training is defined %}
|
| 34 |
{%- set is_training = false %}
|
| 35 |
{%- endif %}
|
| 36 |
-
|
|
|
|
| 37 |
{%- set reasoning_effort = 'no_think' %}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 38 |
{%- endif %}
|
| 39 |
|
| 40 |
{%- macro visible_text(content) -%}
|
|
@@ -112,22 +130,29 @@
|
|
| 112 |
{%- set prev_ns.is_tool = false %}
|
| 113 |
{%- endif %}
|
| 114 |
{%- if message['role'] == 'assistant' %}
|
| 115 |
-
{%- if 'reasoning_content' in message and message['reasoning_content'] is string %}
|
| 116 |
-
{%- set rc = message['reasoning_content'] %}
|
| 117 |
-
{%- elif 'reasoning' in message and message['reasoning'] is string %}
|
| 118 |
-
{%- set rc = message['reasoning'] %}
|
| 119 |
-
{%- else %}
|
| 120 |
-
{%- set rc = none %}
|
| 121 |
-
{%- endif %}
|
| 122 |
{%- if is_training %}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 123 |
{%- if rc is not none %}
|
| 124 |
{%- set content = think_begin_token + rc + think_end_token + visible_text(message['content']) %}
|
| 125 |
{%- else %}
|
| 126 |
{%- set content = think_begin_token + think_end_token + visible_text(message['content']) %}
|
| 127 |
{%- endif %}
|
| 128 |
{%- else %}
|
| 129 |
-
{%- if
|
| 130 |
-
{%- if
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 131 |
{%- set content = think_begin_token + rc + think_end_token + visible_text(message['content']) %}
|
| 132 |
{%- else %}
|
| 133 |
{%- set content = think_begin_token + think_end_token + visible_text(message['content']) %}
|
|
@@ -158,7 +183,9 @@
|
|
| 158 |
{%- endfor %}
|
| 159 |
{{- toolcalls_end_token + eos_token }}
|
| 160 |
{%- else %}
|
| 161 |
-
{%- if
|
|
|
|
|
|
|
| 162 |
{{- content + eos_token }}
|
| 163 |
{%- else %}
|
| 164 |
{{- content }}
|
|
|
|
| 1 |
{#- ----------‑‑‑ special token variables ‑‑‑---------- -#}
|
| 2 |
+
{%- set HYTK = ':opensource' %}
|
| 3 |
+
{%- set eos_token = '<｜hy_eos{}｜>'.format(HYTK) %}
|
| 4 |
+
{%- set bos_token = '<｜hy_begin_of_sentence{}｜>'.format(HYTK) %}
|
| 5 |
+
{%- set pad_token = '<｜hy_pad{}｜>'.format(HYTK) %}
|
| 6 |
+
{%- set user_token = '<｜hy_User{}｜>'.format(HYTK) %}
|
| 7 |
+
{%- set assistant_token = '<｜hy_Assistant{}｜>'.format(HYTK) %}
|
| 8 |
+
{%- set think_begin_token = '<think{}>'.format(HYTK) %}
|
| 9 |
+
{%- set think_end_token = '</think{}>'.format(HYTK) %}
|
| 10 |
+
{%- set toolcalls_begin_token = '<tool_calls{}>'.format(HYTK) %}
|
| 11 |
+
{%- set toolcalls_end_token = '</tool_calls{}>'.format(HYTK) %}
|
| 12 |
+
{%- set toolcall_begin_token = '<tool_call{}>'.format(HYTK) %}
|
| 13 |
+
{%- set toolcall_end_token = '</tool_call{}>'.format(HYTK) %}
|
| 14 |
+
{%- set toolsep_token = '<tool_sep{}>'.format(HYTK) %}
|
| 15 |
+
{%- set argkey_begin_token = '<arg_key{}>'.format(HYTK) %}
|
| 16 |
+
{%- set argkey_end_token = '</arg_key{}>'.format(HYTK) %}
|
| 17 |
+
{%- set argvalue_begin_token = '<arg_value{}>'.format(HYTK) %}
|
| 18 |
+
{%- set argvalue_end_token = '</arg_value{}>'.format(HYTK) %}
|
| 19 |
+
{%- set toolresponses_begin_token = '<tool_responses{}>'.format(HYTK) %}
|
| 20 |
+
{%- set toolresponses_end_token = '</tool_responses{}>'.format(HYTK) %}
|
| 21 |
+
{%- set toolresponse_begin_token = '<tool_response{}>'.format(HYTK) %}
|
| 22 |
+
{%- set toolresponse_end_token = '</tool_response{}>'.format(HYTK) %}
|
| 23 |
+
{%- set reasoning_mode_token = '<｜reasoning_mode{}｜>'.format(HYTK) %}
|
| 24 |
+
|
| 25 |
{#- ----------‑‑‑ hyperparameters variables ‑‑‑---------- -#}
|
| 26 |
{%- if not add_generation_prompt is defined %}
|
| 27 |
{%- set add_generation_prompt = false %}
|
| 28 |
{%- endif %}
|
| 29 |
+
{%- if not preserved_thinking is defined %}
|
| 30 |
+
{%- if not tools %}
|
| 31 |
+
{%- set preserved_thinking = false %}
|
| 32 |
+
{%- else %}
|
| 33 |
+
{%- set preserved_thinking = true %}
|
| 34 |
+
{%- endif %}
|
| 35 |
{%- endif %}
|
| 36 |
{%- if not is_training is defined %}
|
| 37 |
{%- set is_training = false %}
|
| 38 |
{%- endif %}
|
| 39 |
+
|
| 40 |
+
{%- if not reasoning_effort is defined %}
|
| 41 |
{%- set reasoning_effort = 'no_think' %}
|
| 42 |
+
{%- elif reasoning_effort not in ['high', 'low', 'no_think'] %}
|
| 43 |
+
{%- if reasoning_effort is none %}
|
| 44 |
+
{{- raise_exception('reasoning_effort error : None, should be no_think/low/high') }}
|
| 45 |
+
{%- else %}
|
| 46 |
+
{{- raise_exception('reasoning_effort error : ' + reasoning_effort + ', should be no_think/low/high') }}
|
| 47 |
+
{%- endif %}
|
| 48 |
+
{%- endif %}
|
| 49 |
+
|
| 50 |
+
{%- if fallback_strategy is defined and fallback_strategy == 'reasoning_toolcall_retry' %}
|
| 51 |
+
{%- set reasoning_effort = 'high' %}
|
| 52 |
+
{%- set add_generation_prompt = false %}
|
| 53 |
+
{%- endif %}
|
| 54 |
+
{%- if not raw_last_assistant is defined %}
|
| 55 |
+
{%- set raw_last_assistant = false %}
|
| 56 |
{%- endif %}
|
| 57 |
|
| 58 |
{%- macro visible_text(content) -%}
|
|
|
|
| 130 |
{%- set prev_ns.is_tool = false %}
|
| 131 |
{%- endif %}
|
| 132 |
{%- if message['role'] == 'assistant' %}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 133 |
{%- if is_training %}
|
| 134 |
+
{%- if 'reasoning_content' in message and message['reasoning_content'] is string %}
|
| 135 |
+
{%- set rc = message['reasoning_content'] %}
|
| 136 |
+
{%- elif 'reasoning' in message and message['reasoning'] is string %}
|
| 137 |
+
{%- set rc = message['reasoning'] %}
|
| 138 |
+
{%- else %}
|
| 139 |
+
{%- set rc = none %}
|
| 140 |
+
{%- endif %}
|
| 141 |
{%- if rc is not none %}
|
| 142 |
{%- set content = think_begin_token + rc + think_end_token + visible_text(message['content']) %}
|
| 143 |
{%- else %}
|
| 144 |
{%- set content = think_begin_token + think_end_token + visible_text(message['content']) %}
|
| 145 |
{%- endif %}
|
| 146 |
{%- else %}
|
| 147 |
+
{%- if ((preserved_thinking is defined and preserved_thinking) or loop.index0 > ns.last_user_index) %}
|
| 148 |
+
{%- if 'reasoning_content' in message and message['reasoning_content'] is string %}
|
| 149 |
+
{%- set rc = message['reasoning_content'] %}
|
| 150 |
+
{%- elif 'reasoning' in message and message['reasoning'] is string %}
|
| 151 |
+
{%- set rc = message['reasoning'] %}
|
| 152 |
+
{%- else %}
|
| 153 |
+
{%- set rc = none %}
|
| 154 |
+
{%- endif %}
|
| 155 |
+
{%- if rc is not none %}
|
| 156 |
{%- set content = think_begin_token + rc + think_end_token + visible_text(message['content']) %}
|
| 157 |
{%- else %}
|
| 158 |
{%- set content = think_begin_token + think_end_token + visible_text(message['content']) %}
|
|
|
|
| 183 |
{%- endfor %}
|
| 184 |
{{- toolcalls_end_token + eos_token }}
|
| 185 |
{%- else %}
|
| 186 |
+
{%- if loop.last and raw_last_assistant %}
|
| 187 |
+
{{- visible_text(message['content']) }}
|
| 188 |
+
{%- elif not loop.last or is_training %}
|
| 189 |
{{- content + eos_token }}
|
| 190 |
{%- else %}
|
| 191 |
{{- content }}
|
config.json
CHANGED
|
@@ -49,7 +49,7 @@
|
|
| 49 |
"router_scaling_factor": 2.826,
|
| 50 |
"sep_token_id": 120007,
|
| 51 |
"tie_word_embeddings": false,
|
| 52 |
-
"transformers_version": "5.
|
| 53 |
"use_cache": true,
|
| 54 |
"use_grouped_mm": false,
|
| 55 |
"vocab_size": 120832
|
|
|
|
| 49 |
"router_scaling_factor": 2.826,
|
| 50 |
"sep_token_id": 120007,
|
| 51 |
"tie_word_embeddings": false,
|
| 52 |
+
"transformers_version": "5.13.0",
|
| 53 |
"use_cache": true,
|
| 54 |
"use_grouped_mm": false,
|
| 55 |
"vocab_size": 120832
|
generation_config.json
CHANGED
|
@@ -6,6 +6,6 @@
|
|
| 6 |
"temperature": 0.9,
|
| 7 |
"top_k": 40,
|
| 8 |
"top_p": 1,
|
| 9 |
-
"transformers_version": "5.
|
| 10 |
"trust_remote_code": true
|
| 11 |
}
|
|
|
|
| 6 |
"temperature": 0.9,
|
| 7 |
"top_k": 40,
|
| 8 |
"top_p": 1,
|
| 9 |
+
"transformers_version": "5.13.0",
|
| 10 |
"trust_remote_code": true
|
| 11 |
}
|
tokenizer.json
CHANGED
|
@@ -5,7 +5,7 @@
|
|
| 5 |
"added_tokens": [
|
| 6 |
{
|
| 7 |
"id": 120000,
|
| 8 |
-
"content": "<｜
|
| 9 |
"single_word": false,
|
| 10 |
"lstrip": false,
|
| 11 |
"rstrip": false,
|
|
@@ -23,7 +23,7 @@
|
|
| 23 |
},
|
| 24 |
{
|
| 25 |
"id": 120002,
|
| 26 |
-
"content": "<｜
|
| 27 |
"single_word": false,
|
| 28 |
"lstrip": false,
|
| 29 |
"rstrip": false,
|
|
@@ -59,7 +59,7 @@
|
|
| 59 |
},
|
| 60 |
{
|
| 61 |
"id": 120006,
|
| 62 |
-
"content": "<｜hy_User｜>",
|
| 63 |
"single_word": false,
|
| 64 |
"lstrip": false,
|
| 65 |
"rstrip": false,
|
|
@@ -68,7 +68,7 @@
|
|
| 68 |
},
|
| 69 |
{
|
| 70 |
"id": 120007,
|
| 71 |
-
"content": "<｜hy_Assistant｜>",
|
| 72 |
"single_word": false,
|
| 73 |
"lstrip": false,
|
| 74 |
"rstrip": false,
|
|
@@ -230,7 +230,7 @@
|
|
| 230 |
},
|
| 231 |
{
|
| 232 |
"id": 120025,
|
| 233 |
-
"content": "<｜hy_eos｜>",
|
| 234 |
"single_word": false,
|
| 235 |
"lstrip": false,
|
| 236 |
"rstrip": false,
|
|
@@ -266,7 +266,7 @@
|
|
| 266 |
},
|
| 267 |
{
|
| 268 |
"id": 120029,
|
| 269 |
-
"content": "<think>",
|
| 270 |
"single_word": false,
|
| 271 |
"lstrip": false,
|
| 272 |
"rstrip": false,
|
|
@@ -275,7 +275,7 @@
|
|
| 275 |
},
|
| 276 |
{
|
| 277 |
"id": 120030,
|
| 278 |
-
"content": "</think>",
|
| 279 |
"single_word": false,
|
| 280 |
"lstrip": false,
|
| 281 |
"rstrip": false,
|
|
@@ -302,7 +302,7 @@
|
|
| 302 |
},
|
| 303 |
{
|
| 304 |
"id": 120033,
|
| 305 |
-
"content": "<tool_call>",
|
| 306 |
"single_word": false,
|
| 307 |
"lstrip": false,
|
| 308 |
"rstrip": false,
|
|
@@ -311,7 +311,7 @@
|
|
| 311 |
},
|
| 312 |
{
|
| 313 |
"id": 120034,
|
| 314 |
-
"content": "</tool_call>",
|
| 315 |
"single_word": false,
|
| 316 |
"lstrip": false,
|
| 317 |
"rstrip": false,
|
|
@@ -320,7 +320,7 @@
|
|
| 320 |
},
|
| 321 |
{
|
| 322 |
"id": 120035,
|
| 323 |
-
"content": "<tool_calls>",
|
| 324 |
"single_word": false,
|
| 325 |
"lstrip": false,
|
| 326 |
"rstrip": false,
|
|
@@ -329,7 +329,7 @@
|
|
| 329 |
},
|
| 330 |
{
|
| 331 |
"id": 120036,
|
| 332 |
-
"content": "</tool_calls>",
|
| 333 |
"single_word": false,
|
| 334 |
"lstrip": false,
|
| 335 |
"rstrip": false,
|
|
@@ -338,7 +338,7 @@
|
|
| 338 |
},
|
| 339 |
{
|
| 340 |
"id": 120037,
|
| 341 |
-
"content": "<tool_response>",
|
| 342 |
"single_word": false,
|
| 343 |
"lstrip": false,
|
| 344 |
"rstrip": false,
|
|
@@ -347,7 +347,7 @@
|
|
| 347 |
},
|
| 348 |
{
|
| 349 |
"id": 120038,
|
| 350 |
-
"content": "</tool_response>",
|
| 351 |
"single_word": false,
|
| 352 |
"lstrip": false,
|
| 353 |
"rstrip": false,
|
|
@@ -374,7 +374,7 @@
|
|
| 374 |
},
|
| 375 |
{
|
| 376 |
"id": 120041,
|
| 377 |
-
"content": "<tool_responses>",
|
| 378 |
"single_word": false,
|
| 379 |
"lstrip": false,
|
| 380 |
"rstrip": false,
|
|
@@ -383,7 +383,7 @@
|
|
| 383 |
},
|
| 384 |
{
|
| 385 |
"id": 120042,
|
| 386 |
-
"content": "</tool_responses>",
|
| 387 |
"single_word": false,
|
| 388 |
"lstrip": false,
|
| 389 |
"rstrip": false,
|
|
@@ -392,7 +392,7 @@
|
|
| 392 |
},
|
| 393 |
{
|
| 394 |
"id": 120043,
|
| 395 |
-
"content": "<tool_sep>",
|
| 396 |
"single_word": false,
|
| 397 |
"lstrip": false,
|
| 398 |
"rstrip": false,
|
|
@@ -401,7 +401,7 @@
|
|
| 401 |
},
|
| 402 |
{
|
| 403 |
"id": 120044,
|
| 404 |
-
"content": "<｜reasoning_mode｜>",
|
| 405 |
"single_word": false,
|
| 406 |
"lstrip": false,
|
| 407 |
"rstrip": false,
|
|
@@ -410,7 +410,7 @@
|
|
| 410 |
},
|
| 411 |
{
|
| 412 |
"id": 120045,
|
| 413 |
-
"content": "<｜tool▁calls▁begin｜>",
|
| 414 |
"single_word": false,
|
| 415 |
"lstrip": false,
|
| 416 |
"rstrip": false,
|
|
@@ -419,7 +419,7 @@
|
|
| 419 |
},
|
| 420 |
{
|
| 421 |
"id": 120046,
|
| 422 |
-
"content": "<｜tool▁calls▁end｜>",
|
| 423 |
"single_word": false,
|
| 424 |
"lstrip": false,
|
| 425 |
"rstrip": false,
|
|
@@ -428,7 +428,7 @@
|
|
| 428 |
},
|
| 429 |
{
|
| 430 |
"id": 120047,
|
| 431 |
-
"content": "<｜tool▁call▁begin｜>",
|
| 432 |
"single_word": false,
|
| 433 |
"lstrip": false,
|
| 434 |
"rstrip": false,
|
|
@@ -437,7 +437,7 @@
|
|
| 437 |
},
|
| 438 |
{
|
| 439 |
"id": 120048,
|
| 440 |
-
"content": "<｜tool▁call▁end｜>",
|
| 441 |
"single_word": false,
|
| 442 |
"lstrip": false,
|
| 443 |
"rstrip": false,
|
|
@@ -446,7 +446,7 @@
|
|
| 446 |
},
|
| 447 |
{
|
| 448 |
"id": 120049,
|
| 449 |
-
"content": "<｜tool▁sep｜>",
|
| 450 |
"single_word": false,
|
| 451 |
"lstrip": false,
|
| 452 |
"rstrip": false,
|
|
@@ -455,7 +455,7 @@
|
|
| 455 |
},
|
| 456 |
{
|
| 457 |
"id": 120050,
|
| 458 |
-
"content": "<｜tool▁output▁begin｜>",
|
| 459 |
"single_word": false,
|
| 460 |
"lstrip": false,
|
| 461 |
"rstrip": false,
|
|
@@ -464,7 +464,7 @@
|
|
| 464 |
},
|
| 465 |
{
|
| 466 |
"id": 120051,
|
| 467 |
-
"content": "<｜tool▁output▁end｜>",
|
| 468 |
"single_word": false,
|
| 469 |
"lstrip": false,
|
| 470 |
"rstrip": false,
|
|
@@ -599,7 +599,7 @@
|
|
| 599 |
},
|
| 600 |
{
|
| 601 |
"id": 120066,
|
| 602 |
-
"content": "<arg_key>",
|
| 603 |
"single_word": false,
|
| 604 |
"lstrip": false,
|
| 605 |
"rstrip": false,
|
|
@@ -608,7 +608,7 @@
|
|
| 608 |
},
|
| 609 |
{
|
| 610 |
"id": 120067,
|
| 611 |
-
"content": "</arg_key>",
|
| 612 |
"single_word": false,
|
| 613 |
"lstrip": false,
|
| 614 |
"rstrip": false,
|
|
@@ -617,7 +617,7 @@
|
|
| 617 |
},
|
| 618 |
{
|
| 619 |
"id": 120068,
|
| 620 |
-
"content": "<arg_value>",
|
| 621 |
"single_word": false,
|
| 622 |
"lstrip": false,
|
| 623 |
"rstrip": false,
|
|
@@ -626,7 +626,7 @@
|
|
| 626 |
},
|
| 627 |
{
|
| 628 |
"id": 120069,
|
| 629 |
-
"content": "</arg_value>",
|
| 630 |
"single_word": false,
|
| 631 |
"lstrip": false,
|
| 632 |
"rstrip": false,
|
|
|
|
| 5 |
"added_tokens": [
|
| 6 |
{
|
| 7 |
"id": 120000,
|
| 8 |
+
"content": "<｜hy_begin_of_sentence:opensource｜>",
|
| 9 |
"single_word": false,
|
| 10 |
"lstrip": false,
|
| 11 |
"rstrip": false,
|
|
|
|
| 23 |
},
|
| 24 |
{
|
| 25 |
"id": 120002,
|
| 26 |
+
"content": "<｜hy_pad:opensource｜>",
|
| 27 |
"single_word": false,
|
| 28 |
"lstrip": false,
|
| 29 |
"rstrip": false,
|
|
|
|
| 59 |
},
|
| 60 |
{
|
| 61 |
"id": 120006,
|
| 62 |
+
"content": "<｜hy_User:opensource｜>",
|
| 63 |
"single_word": false,
|
| 64 |
"lstrip": false,
|
| 65 |
"rstrip": false,
|
|
|
|
| 68 |
},
|
| 69 |
{
|
| 70 |
"id": 120007,
|
| 71 |
+
"content": "<｜hy_Assistant:opensource｜>",
|
| 72 |
"single_word": false,
|
| 73 |
"lstrip": false,
|
| 74 |
"rstrip": false,
|
|
|
|
| 230 |
},
|
| 231 |
{
|
| 232 |
"id": 120025,
|
| 233 |
+
"content": "<｜hy_eos:opensource｜>",
|
| 234 |
"single_word": false,
|
| 235 |
"lstrip": false,
|
| 236 |
"rstrip": false,
|
|
|
|
| 266 |
},
|
| 267 |
{
|
| 268 |
"id": 120029,
|
| 269 |
+
"content": "<think:opensource>",
|
| 270 |
"single_word": false,
|
| 271 |
"lstrip": false,
|
| 272 |
"rstrip": false,
|
|
|
|
| 275 |
},
|
| 276 |
{
|
| 277 |
"id": 120030,
|
| 278 |
+
"content": "</think:opensource>",
|
| 279 |
"single_word": false,
|
| 280 |
"lstrip": false,
|
| 281 |
"rstrip": false,
|
|
|
|
| 302 |
},
|
| 303 |
{
|
| 304 |
"id": 120033,
|
| 305 |
+
"content": "<tool_call:opensource>",
|
| 306 |
"single_word": false,
|
| 307 |
"lstrip": false,
|
| 308 |
"rstrip": false,
|
|
|
|
| 311 |
},
|
| 312 |
{
|
| 313 |
"id": 120034,
|
| 314 |
+
"content": "</tool_call:opensource>",
|
| 315 |
"single_word": false,
|
| 316 |
"lstrip": false,
|
| 317 |
"rstrip": false,
|
|
|
|
| 320 |
},
|
| 321 |
{
|
| 322 |
"id": 120035,
|
| 323 |
+
"content": "<tool_calls:opensource>",
|
| 324 |
"single_word": false,
|
| 325 |
"lstrip": false,
|
| 326 |
"rstrip": false,
|
|
|
|
| 329 |
},
|
| 330 |
{
|
| 331 |
"id": 120036,
|
| 332 |
+
"content": "</tool_calls:opensource>",
|
| 333 |
"single_word": false,
|
| 334 |
"lstrip": false,
|
| 335 |
"rstrip": false,
|
|
|
|
| 338 |
},
|
| 339 |
{
|
| 340 |
"id": 120037,
|
| 341 |
+
"content": "<tool_response:opensource>",
|
| 342 |
"single_word": false,
|
| 343 |
"lstrip": false,
|
| 344 |
"rstrip": false,
|
|
|
|
| 347 |
},
|
| 348 |
{
|
| 349 |
"id": 120038,
|
| 350 |
+
"content": "</tool_response:opensource>",
|
| 351 |
"single_word": false,
|
| 352 |
"lstrip": false,
|
| 353 |
"rstrip": false,
|
|
|
|
| 374 |
},
|
| 375 |
{
|
| 376 |
"id": 120041,
|
| 377 |
+
"content": "<tool_responses:opensource>",
|
| 378 |
"single_word": false,
|
| 379 |
"lstrip": false,
|
| 380 |
"rstrip": false,
|
|
|
|
| 383 |
},
|
| 384 |
{
|
| 385 |
"id": 120042,
|
| 386 |
+
"content": "</tool_responses:opensource>",
|
| 387 |
"single_word": false,
|
| 388 |
"lstrip": false,
|
| 389 |
"rstrip": false,
|
|
|
|
| 392 |
},
|
| 393 |
{
|
| 394 |
"id": 120043,
|
| 395 |
+
"content": "<tool_sep:opensource>",
|
| 396 |
"single_word": false,
|
| 397 |
"lstrip": false,
|
| 398 |
"rstrip": false,
|
|
|
|
| 401 |
},
|
| 402 |
{
|
| 403 |
"id": 120044,
|
| 404 |
+
"content": "<｜reasoning_mode:opensource｜>",
|
| 405 |
"single_word": false,
|
| 406 |
"lstrip": false,
|
| 407 |
"rstrip": false,
|
|
|
|
| 410 |
},
|
| 411 |
{
|
| 412 |
"id": 120045,
|
| 413 |
+
"content": "<｜tool▁calls▁begin:opensource｜>",
|
| 414 |
"single_word": false,
|
| 415 |
"lstrip": false,
|
| 416 |
"rstrip": false,
|
|
|
|
| 419 |
},
|
| 420 |
{
|
| 421 |
"id": 120046,
|
| 422 |
+
"content": "<｜tool▁calls▁end:opensource｜>",
|
| 423 |
"single_word": false,
|
| 424 |
"lstrip": false,
|
| 425 |
"rstrip": false,
|
|
|
|
| 428 |
},
|
| 429 |
{
|
| 430 |
"id": 120047,
|
| 431 |
+
"content": "<｜tool▁call▁begin:opensource｜>",
|
| 432 |
"single_word": false,
|
| 433 |
"lstrip": false,
|
| 434 |
"rstrip": false,
|
|
|
|
| 437 |
},
|
| 438 |
{
|
| 439 |
"id": 120048,
|
| 440 |
+
"content": "<｜tool▁call▁end:opensource｜>",
|
| 441 |
"single_word": false,
|
| 442 |
"lstrip": false,
|
| 443 |
"rstrip": false,
|
|
|
|
| 446 |
},
|
| 447 |
{
|
| 448 |
"id": 120049,
|
| 449 |
+
"content": "<｜tool▁sep:opensource｜>",
|
| 450 |
"single_word": false,
|
| 451 |
"lstrip": false,
|
| 452 |
"rstrip": false,
|
|
|
|
| 455 |
},
|
| 456 |
{
|
| 457 |
"id": 120050,
|
| 458 |
+
"content": "<｜tool▁output▁begin:opensource｜>",
|
| 459 |
"single_word": false,
|
| 460 |
"lstrip": false,
|
| 461 |
"rstrip": false,
|
|
|
|
| 464 |
},
|
| 465 |
{
|
| 466 |
"id": 120051,
|
| 467 |
+
"content": "<｜tool▁output▁end:opensource｜>",
|
| 468 |
"single_word": false,
|
| 469 |
"lstrip": false,
|
| 470 |
"rstrip": false,
|
|
|
|
| 599 |
},
|
| 600 |
{
|
| 601 |
"id": 120066,
|
| 602 |
+
"content": "<arg_key:opensource>",
|
| 603 |
"single_word": false,
|
| 604 |
"lstrip": false,
|
| 605 |
"rstrip": false,
|
|
|
|
| 608 |
},
|
| 609 |
{
|
| 610 |
"id": 120067,
|
| 611 |
+
"content": "</arg_key:opensource>",
|
| 612 |
"single_word": false,
|
| 613 |
"lstrip": false,
|
| 614 |
"rstrip": false,
|
|
|
|
| 617 |
},
|
| 618 |
{
|
| 619 |
"id": 120068,
|
| 620 |
+
"content": "<arg_value:opensource>",
|
| 621 |
"single_word": false,
|
| 622 |
"lstrip": false,
|
| 623 |
"rstrip": false,
|
|
|
|
| 626 |
},
|
| 627 |
{
|
| 628 |
"id": 120069,
|
| 629 |
+
"content": "</arg_value:opensource>",
|
| 630 |
"single_word": false,
|
| 631 |
"lstrip": false,
|
| 632 |
"rstrip": false,
|
tokenizer_config.json
CHANGED
|
@@ -1,11 +1,12 @@
|
|
| 1 |
{
|
| 2 |
"backend": "tokenizers",
|
| 3 |
-
"bos_token": "<｜
|
| 4 |
"clean_up_tokenization_spaces": false,
|
| 5 |
-
"eos_token": "<｜hy_eos｜>",
|
| 6 |
"is_local": false,
|
| 7 |
"local_files_only": false,
|
| 8 |
"model_max_length": 1000000000000000019884624838656,
|
| 9 |
-
"pad_token": "<｜
|
|
|
|
| 10 |
"tokenizer_class": "TokenizersBackend"
|
| 11 |
}
|
|
|
|
| 1 |
{
|
| 2 |
"backend": "tokenizers",
|
| 3 |
+
"bos_token": "<｜hy_begin_of_sentence:opensource｜>",
|
| 4 |
"clean_up_tokenization_spaces": false,
|
| 5 |
+
"eos_token": "<｜hy_eos:opensource｜>",
|
| 6 |
"is_local": false,
|
| 7 |
"local_files_only": false,
|
| 8 |
"model_max_length": 1000000000000000019884624838656,
|
| 9 |
+
"pad_token": "<｜hy_pad:opensource｜>",
|
| 10 |
+
"token_suffix": ":opensource",
|
| 11 |
"tokenizer_class": "TokenizersBackend"
|
| 12 |
}
|