final
Browse files- .gitattributes +3 -0
- verifier-qwen3.5-2b/README.md +207 -0
- verifier-qwen3.5-2b/adapter_config.json +56 -0
- verifier-qwen3.5-2b/adapter_model.safetensors +3 -0
- verifier-qwen3.5-2b/calibration_predictions.jsonl +99 -0
- verifier-qwen3.5-2b/chat_template.jinja +154 -0
- verifier-qwen3.5-2b/tb/events.out.tfevents.1790667609.02b48645612f.2184.0 +2 -2
- verifier-qwen3.5-2b/test_eval/baselines.json +227 -0
- verifier-qwen3.5-2b/test_eval/comparison.md +34 -0
- verifier-qwen3.5-2b/test_eval/metrics.json +25 -0
- verifier-qwen3.5-2b/test_eval/predictions.jsonl +3 -0
- verifier-qwen3.5-2b/test_eval_entity/baselines.json +227 -0
- verifier-qwen3.5-2b/test_eval_entity/metrics.json +25 -0
- verifier-qwen3.5-2b/test_eval_entity/predictions.jsonl +3 -0
- verifier-qwen3.5-2b/test_eval_typo/baselines.json +227 -0
- verifier-qwen3.5-2b/test_eval_typo/metrics.json +25 -0
- verifier-qwen3.5-2b/test_eval_typo/predictions.jsonl +0 -0
- verifier-qwen3.5-2b/tokenizer.json +3 -0
- verifier-qwen3.5-2b/tokenizer_config.json +32 -0
- verifier-qwen3.5-2b/training_args.bin +3 -0
- verifier-qwen3.5-2b/validation_predictions.jsonl +0 -0
- verifier-qwen3.5-2b/verifier_scores.json +88 -0
.gitattributes
CHANGED
|
@@ -39,3 +39,6 @@ qwen3.5-2b-50pct-masked-weighted/tokenizer.json filter=lfs diff=lfs merge=lfs -t
|
|
| 39 |
edit-qwen3.5-2b/test_eval/predictions.jsonl filter=lfs diff=lfs merge=lfs -text
|
| 40 |
edit-qwen3.5-2b/tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
| 41 |
qwen3.5-2b-100pct-masked-weighted-v2/tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
| 39 |
edit-qwen3.5-2b/test_eval/predictions.jsonl filter=lfs diff=lfs merge=lfs -text
|
| 40 |
edit-qwen3.5-2b/tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
| 41 |
qwen3.5-2b-100pct-masked-weighted-v2/tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
| 42 |
+
verifier-qwen3.5-2b/test_eval/predictions.jsonl filter=lfs diff=lfs merge=lfs -text
|
| 43 |
+
verifier-qwen3.5-2b/test_eval_entity/predictions.jsonl filter=lfs diff=lfs merge=lfs -text
|
| 44 |
+
verifier-qwen3.5-2b/tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
verifier-qwen3.5-2b/README.md
ADDED
|
@@ -0,0 +1,207 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
base_model: Qwen/Qwen3.5-2B
|
| 3 |
+
library_name: peft
|
| 4 |
+
pipeline_tag: text-generation
|
| 5 |
+
tags:
|
| 6 |
+
- base_model:adapter:Qwen/Qwen3.5-2B
|
| 7 |
+
- lora
|
| 8 |
+
- transformers
|
| 9 |
+
---
|
| 10 |
+
|
| 11 |
+
# Model Card for Model ID
|
| 12 |
+
|
| 13 |
+
<!-- Provide a quick summary of what the model is/does. -->
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
## Model Details
|
| 18 |
+
|
| 19 |
+
### Model Description
|
| 20 |
+
|
| 21 |
+
<!-- Provide a longer summary of what this model is. -->
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
- **Developed by:** [More Information Needed]
|
| 26 |
+
- **Funded by [optional]:** [More Information Needed]
|
| 27 |
+
- **Shared by [optional]:** [More Information Needed]
|
| 28 |
+
- **Model type:** [More Information Needed]
|
| 29 |
+
- **Language(s) (NLP):** [More Information Needed]
|
| 30 |
+
- **License:** [More Information Needed]
|
| 31 |
+
- **Finetuned from model [optional]:** [More Information Needed]
|
| 32 |
+
|
| 33 |
+
### Model Sources [optional]
|
| 34 |
+
|
| 35 |
+
<!-- Provide the basic links for the model. -->
|
| 36 |
+
|
| 37 |
+
- **Repository:** [More Information Needed]
|
| 38 |
+
- **Paper [optional]:** [More Information Needed]
|
| 39 |
+
- **Demo [optional]:** [More Information Needed]
|
| 40 |
+
|
| 41 |
+
## Uses
|
| 42 |
+
|
| 43 |
+
<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
|
| 44 |
+
|
| 45 |
+
### Direct Use
|
| 46 |
+
|
| 47 |
+
<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
|
| 48 |
+
|
| 49 |
+
[More Information Needed]
|
| 50 |
+
|
| 51 |
+
### Downstream Use [optional]
|
| 52 |
+
|
| 53 |
+
<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
|
| 54 |
+
|
| 55 |
+
[More Information Needed]
|
| 56 |
+
|
| 57 |
+
### Out-of-Scope Use
|
| 58 |
+
|
| 59 |
+
<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
|
| 60 |
+
|
| 61 |
+
[More Information Needed]
|
| 62 |
+
|
| 63 |
+
## Bias, Risks, and Limitations
|
| 64 |
+
|
| 65 |
+
<!-- This section is meant to convey both technical and sociotechnical limitations. -->
|
| 66 |
+
|
| 67 |
+
[More Information Needed]
|
| 68 |
+
|
| 69 |
+
### Recommendations
|
| 70 |
+
|
| 71 |
+
<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
|
| 72 |
+
|
| 73 |
+
Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
|
| 74 |
+
|
| 75 |
+
## How to Get Started with the Model
|
| 76 |
+
|
| 77 |
+
Use the code below to get started with the model.
|
| 78 |
+
|
| 79 |
+
[More Information Needed]
|
| 80 |
+
|
| 81 |
+
## Training Details
|
| 82 |
+
|
| 83 |
+
### Training Data
|
| 84 |
+
|
| 85 |
+
<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
|
| 86 |
+
|
| 87 |
+
[More Information Needed]
|
| 88 |
+
|
| 89 |
+
### Training Procedure
|
| 90 |
+
|
| 91 |
+
<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
|
| 92 |
+
|
| 93 |
+
#### Preprocessing [optional]
|
| 94 |
+
|
| 95 |
+
[More Information Needed]
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
#### Training Hyperparameters
|
| 99 |
+
|
| 100 |
+
- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
|
| 101 |
+
|
| 102 |
+
#### Speeds, Sizes, Times [optional]
|
| 103 |
+
|
| 104 |
+
<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
|
| 105 |
+
|
| 106 |
+
[More Information Needed]
|
| 107 |
+
|
| 108 |
+
## Evaluation
|
| 109 |
+
|
| 110 |
+
<!-- This section describes the evaluation protocols and provides the results. -->
|
| 111 |
+
|
| 112 |
+
### Testing Data, Factors & Metrics
|
| 113 |
+
|
| 114 |
+
#### Testing Data
|
| 115 |
+
|
| 116 |
+
<!-- This should link to a Dataset Card if possible. -->
|
| 117 |
+
|
| 118 |
+
[More Information Needed]
|
| 119 |
+
|
| 120 |
+
#### Factors
|
| 121 |
+
|
| 122 |
+
<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
|
| 123 |
+
|
| 124 |
+
[More Information Needed]
|
| 125 |
+
|
| 126 |
+
#### Metrics
|
| 127 |
+
|
| 128 |
+
<!-- These are the evaluation metrics being used, ideally with a description of why. -->
|
| 129 |
+
|
| 130 |
+
[More Information Needed]
|
| 131 |
+
|
| 132 |
+
### Results
|
| 133 |
+
|
| 134 |
+
[More Information Needed]
|
| 135 |
+
|
| 136 |
+
#### Summary
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
## Model Examination [optional]
|
| 141 |
+
|
| 142 |
+
<!-- Relevant interpretability work for the model goes here -->
|
| 143 |
+
|
| 144 |
+
[More Information Needed]
|
| 145 |
+
|
| 146 |
+
## Environmental Impact
|
| 147 |
+
|
| 148 |
+
<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
|
| 149 |
+
|
| 150 |
+
Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
|
| 151 |
+
|
| 152 |
+
- **Hardware Type:** [More Information Needed]
|
| 153 |
+
- **Hours used:** [More Information Needed]
|
| 154 |
+
- **Cloud Provider:** [More Information Needed]
|
| 155 |
+
- **Compute Region:** [More Information Needed]
|
| 156 |
+
- **Carbon Emitted:** [More Information Needed]
|
| 157 |
+
|
| 158 |
+
## Technical Specifications [optional]
|
| 159 |
+
|
| 160 |
+
### Model Architecture and Objective
|
| 161 |
+
|
| 162 |
+
[More Information Needed]
|
| 163 |
+
|
| 164 |
+
### Compute Infrastructure
|
| 165 |
+
|
| 166 |
+
[More Information Needed]
|
| 167 |
+
|
| 168 |
+
#### Hardware
|
| 169 |
+
|
| 170 |
+
[More Information Needed]
|
| 171 |
+
|
| 172 |
+
#### Software
|
| 173 |
+
|
| 174 |
+
[More Information Needed]
|
| 175 |
+
|
| 176 |
+
## Citation [optional]
|
| 177 |
+
|
| 178 |
+
<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
|
| 179 |
+
|
| 180 |
+
**BibTeX:**
|
| 181 |
+
|
| 182 |
+
[More Information Needed]
|
| 183 |
+
|
| 184 |
+
**APA:**
|
| 185 |
+
|
| 186 |
+
[More Information Needed]
|
| 187 |
+
|
| 188 |
+
## Glossary [optional]
|
| 189 |
+
|
| 190 |
+
<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
|
| 191 |
+
|
| 192 |
+
[More Information Needed]
|
| 193 |
+
|
| 194 |
+
## More Information [optional]
|
| 195 |
+
|
| 196 |
+
[More Information Needed]
|
| 197 |
+
|
| 198 |
+
## Model Card Authors [optional]
|
| 199 |
+
|
| 200 |
+
[More Information Needed]
|
| 201 |
+
|
| 202 |
+
## Model Card Contact
|
| 203 |
+
|
| 204 |
+
[More Information Needed]
|
| 205 |
+
### Framework versions
|
| 206 |
+
|
| 207 |
+
- PEFT 0.21.0
|
verifier-qwen3.5-2b/adapter_config.json
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"alora_invocation_tokens": null,
|
| 3 |
+
"alpha_pattern": {},
|
| 4 |
+
"arrow_config": null,
|
| 5 |
+
"auto_mapping": null,
|
| 6 |
+
"base_model_name_or_path": "Qwen/Qwen3.5-2B",
|
| 7 |
+
"bias": "none",
|
| 8 |
+
"corda_config": null,
|
| 9 |
+
"ensure_weight_tying": false,
|
| 10 |
+
"eva_config": null,
|
| 11 |
+
"exclude_modules": null,
|
| 12 |
+
"fan_in_fan_out": false,
|
| 13 |
+
"inference_mode": true,
|
| 14 |
+
"init_lora_weights": true,
|
| 15 |
+
"kasa_config": null,
|
| 16 |
+
"layer_replication": null,
|
| 17 |
+
"layers_pattern": null,
|
| 18 |
+
"layers_to_transform": null,
|
| 19 |
+
"loftq_config": {},
|
| 20 |
+
"lora_alpha": 64,
|
| 21 |
+
"lora_bias": false,
|
| 22 |
+
"lora_dropout": 0.05,
|
| 23 |
+
"lora_ga_config": null,
|
| 24 |
+
"megatron_config": null,
|
| 25 |
+
"megatron_core": "megatron.core",
|
| 26 |
+
"modules_to_save": null,
|
| 27 |
+
"monteclora_config": null,
|
| 28 |
+
"peft_type": "LORA",
|
| 29 |
+
"peft_version": "0.21.0",
|
| 30 |
+
"qalora_group_size": 16,
|
| 31 |
+
"r": 32,
|
| 32 |
+
"rank_pattern": {},
|
| 33 |
+
"revision": null,
|
| 34 |
+
"target_modules": [
|
| 35 |
+
"o_proj",
|
| 36 |
+
"q_proj",
|
| 37 |
+
"in_proj_a",
|
| 38 |
+
"v_proj",
|
| 39 |
+
"gate_proj",
|
| 40 |
+
"down_proj",
|
| 41 |
+
"k_proj",
|
| 42 |
+
"in_proj_z",
|
| 43 |
+
"in_proj_b",
|
| 44 |
+
"out_proj",
|
| 45 |
+
"in_proj_qkv",
|
| 46 |
+
"up_proj"
|
| 47 |
+
],
|
| 48 |
+
"target_parameters": null,
|
| 49 |
+
"task_type": "CAUSAL_LM",
|
| 50 |
+
"trainable_token_indices": null,
|
| 51 |
+
"use_bdlora": null,
|
| 52 |
+
"use_dora": false,
|
| 53 |
+
"use_qalora": false,
|
| 54 |
+
"use_rslora": false,
|
| 55 |
+
"velora_config": null
|
| 56 |
+
}
|
verifier-qwen3.5-2b/adapter_model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:fbbb89e4de20fd72514f274abecdf708c6865451b15d904b7203c8c670472c6d
|
| 3 |
+
size 134604152
|
verifier-qwen3.5-2b/calibration_predictions.jsonl
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"id": "c0a9a6d01ef9e9bf", "call_id": "060ef92629471feac0c652c7dfcc67f6c113ed042d373af155ec08e61dcdd612", "stratum": "phonetic_sub", "crm_names": [], "left": "بفرمایید پیگیری میکنیم خدمتتون اطلاعرسانی میشه خواهش میکنم وقت میشه نظرسنجی تماس", "original": "خدانگهدار", "proposed": "خدانگهدارتون", "right": "", "label": "NO", "label_source": "hand label", "p_yes": 0.06008664851469343}
|
| 2 |
+
{"id": "9c68d516cb84e0e3", "call_id": "749145ddce4112888fbfad313d9f26b8badbae7f596a9105d1aedeb4a3306fe2", "stratum": "other_sub", "crm_names": [], "left": "انجام میگیره بنده پیگیری لازمش رو بابت این مورد براتون انجام خواهم", "original": "دادم", "proposed": "داد حتما", "right": "خواهش میکنم سلامت باشین روزتون بخیر نظرسنجیتون میشین خدانگهدار", "label": "YES", "label_source": "hand label", "p_yes": 0.09534945960549933}
|
| 3 |
+
{"id": "0d8bfb63e5d8d41c", "call_id": "091889f7a33fd89e4ba7e603a1e5ff33fc6249f1f63b32dde32a8bb3006eec00", "stratum": "other_sub", "crm_names": [], "left": "بشه که اگر وارد مراحل پردازش بشه کنسلی و بازگشت وجه سه", "original": "الی", "proposed": "تا", "right": "پنج روز زمان بر هست خواهش میکنم ممنون میشم در نظر سنجی", "label": "NO", "label_source": "hand label", "p_yes": 0.5312093828000823}
|
| 4 |
+
{"id": "d34f77565d0f9653", "call_id": "387072344a2eac0a54a54bfbb00a2b7270b2693f435b740fbba2211027d4965b", "stratum": "other_sub", "crm_names": [], "left": "رو واریز میکنیم امروز براتون یعنی دفعهاند واریزی قرار بودیم نه امروز", "original": "دفعهاند", "proposed": "در فرایند", "right": "واریزی قرار میدیم من اشتباه گفتم من مزاحم میخوام تا کمتر از", "label": "YES", "label_source": "hand label", "p_yes": 0.6791786954442651}
|
| 5 |
+
{"id": "85e7d4492f9e768d", "call_id": "38009a9f92106a57eabbefd582315d339af49c990d43eddfb7009804e2edf6dc", "stratum": "phonetic_sub", "crm_names": [], "left": "خودم بردارم هم مشکل ندارم مرسی اگه لطف کنین امروز یه کار", "original": "کنی همکارتون", "proposed": "کنین همکاراتون", "right": "چیز کنن نتیجه رو به من بگن ممنون میشم ازتون ممنونم خداحافظ", "label": "NO", "label_source": "hand label", "p_yes": 0.1824255239642128}
|
| 6 |
+
{"id": "1dc1dd70081eb12d", "call_id": "09e7ee6efeb9a9185ccdee27b0fb6e0bf55cc697b304965139c1871809dd9c9e", "stratum": "entity", "crm_names": [], "left": "بوده مرجوع کردم هنوز نوشته در حال دسترک بررسی هستش به نام", "original": "تنظیری", "proposed": "تنزوری", "right": "بله این دو تاش مرجوع شده بوده میخواستم دوباره بررسی بشه بله", "label": "NO", "label_source": "hand label", "p_yes": 0.3486451556779079}
|
| 7 |
+
{"id": "4acf27c2eb5af3d5", "call_id": "2a255bf31e27594724cd2ee10922150e9193ea920f2e00c2919e47774430549d", "stratum": "entity", "crm_names": [], "left": "این همه اپ هست که کار میکنی به راحتی ادیت میکنی خودت", "original": "کارات", "proposed": "کارهات", "right": "رو انجام میدی یعنی میگی پشتیبانی من از دیروز پشتیبانی هم ثبت", "label": "NO", "label_source": "hand label", "p_yes": 0.3208213109574147}
|
| 8 |
+
{"id": "edd4d350c7958e46", "call_id": "4a1ddb8c59e8c2e87b8040de6f4c8115c4a3ad0946356c4d323c0d49ecf57ed5", "stratum": "phonetic_sub", "crm_names": [], "left": "سفارشی که حذف شده امروز یه خاک گربه بوده کنسل شده و", "original": "با بقیه", "proposed": "مابقی", "right": "هم یه تعداد از آیتمها کیسه همون بوده بانکه مدل چهارگوش سبد", "label": "YES", "label_source": "hand label", "p_yes": 0.5926665897048025}
|
| 9 |
+
{"id": "2bc0809bc8123ba3", "call_id": "6d5a7795b6bcbfcec02509cb851ebded21b52a55cad5bc08a807443eb1562117", "stratum": "other_sub", "crm_names": ["زنده", "سیده", "معصومه"], "left": "بمونید من ثبت کنم از واحد شکایات باهاتون تماس بگیرن مراجعه بالاترمون", "original": "حتما", "proposed": "هستن", "right": "برای پیگیریش من باید هم ثبت میکنم الان من ثبت میکنم که", "label": "YES", "label_source": "hand label", "p_yes": 0.1192029242006422}
|
| 10 |
+
{"id": "b0d0c5f83b551757", "call_id": "53d901213bae56b8fd6c0163194f27de70d14caaf3f8c5e7fd32106beded1bb3", "stratum": "entity", "crm_names": ["بختیاری", "لیلی", "مهدی", "هیلی"], "left": "", "original": "خیلی", "proposed": "هیلی", "right": "هستم بفرمایید سلامت باشین الان براتون چک میکنم با همین شماره تماس", "label": "YES", "label_source": "hand label", "p_yes": 0.3775406736000799}
|
| 11 |
+
{"id": "647299f27c5d1522", "call_id": "021f033588676a6979bde8cc480d2e9bdec7027f75debf31a465c3f7a4149145", "stratum": "phonetic_sub", "crm_names": [], "left": "تو کشیام توی خلیج فارسیم حقیقتش اصلا به این صحبت کنید این", "original": "شمارهی", "proposed": "شماره", "right": "گیرنده که خواهر بندهست رو بهتون بدم که ایشون با یعنی تحویل", "label": "NO", "label_source": "hand label", "p_yes": 0.24508500651500825}
|
| 12 |
+
{"id": "97e018f7c5a6e525", "call_id": "1ee8354b349ac0ad617dfbef4bbe43c0966fe6b3bd9f19bcfe6699725325d0cf", "stratum": "long", "crm_names": [], "left": "ازتون تحویل بگیرن و مجدد بسته پنج عددیاش رو براتون ارسال بکنن", "original": "صورت", "proposed": "الان سفارشی که من میتونم ببینم ازتون یکیش در حال ارسال هستش یکیش هم توسط", "right": "شما دریافت شده آیتمهای تگ انافسی مدل دویست و پانزده بسته پنج", "label": "YES", "label_source": "hand label", "p_yes": 0.7772998548605858}
|
| 13 |
+
{"id": "e5d271e4c6e0b311", "call_id": "342b2d1b0fb5c9944c0f80c3ef09eb9c23532e0f86b9b752a84a35153a378fec", "stratum": "phonetic_sub", "crm_names": [], "left": "پنج اف ای همراه با تاور بود خورداد بوده فکر میکنم هشت", "original": "خورداد هزینهشو", "proposed": "خرداد هزینهاش رو", "right": "بخش نقدیشو دیجیکالا سرویس واسم انتقال داد ولی اقساطم کسر نشده الانم", "label": "YES", "label_source": "hand label", "p_yes": 0.6791786936723664}
|
| 14 |
+
{"id": "77fb33498b3ccc4d", "call_id": "3c19e402064b5299215f00fa4a74224b4fba85ce59404caaa8fffa2d83494cba", "stratum": "other_sub", "crm_names": ["رجایی", "رزاقی", "رضا", "فرناز"], "left": "لطفا بابت روغنتون ارسال سفارش از طریق تیپاکس بوده جناب عزت طلب", "original": "بر", "proposed": "در", "right": "سایت تیپاکس چک کردین موردتون رو ارسال سفارش از طریق تیپاکس بوده", "label": "YES", "label_source": "hand label", "p_yes": 0.40733340597117745}
|
| 15 |
+
{"id": "019df4b23e0fdcd5", "call_id": "0e5791c1ed748fa089db6b3a72b4ba0ee8acec72488bc8f6864cb1bb8cc4a67c", "stratum": "entity", "crm_names": [], "left": "و بیست و یک ثبتش کردیم اگر پشت خط صبوری بفرمایید من", "original": "مغایرتتون", "proposed": "موقعیتتون", "right": "رو برطرف میکنم خیلی لطف میکنید کلا مغایرتتون الان برطرف شده آقای", "label": "NO", "label_source": "hand label", "p_yes": 0.8175744633392594}
|
| 16 |
+
{"id": "9b7411281714aa85", "call_id": "6d980501d6fc3e887b6a68553098e0470cc9fc81753a5fde1be0a17c394ee67c", "stratum": "other_sub", "crm_names": [], "left": "زمانبر شده چک میکنم خدمتتون کالاتون که تامین شده آماده هستش به", "original": "صحبه رشد", "proposed": "شعبه راس", "right": "ارسال هم انجام شده فقط بابت دریافت بستهتون اجازه بفرمایید ما یه", "label": "YES", "label_source": "hand label", "p_yes": 0.8670357542841919}
|
| 17 |
+
{"id": "1210e5ac58e16fcb", "call_id": "128aeb38ddafadd739be684d4db2209e8d436a2a58fbec4494138b9b0d4e3c06", "stratum": "entity", "crm_names": [], "left": "شده مجدد اول شهری ور ماه لطف کردید اشتراک گرفتید میدونم آقای", "original": "سخره", "proposed": "فخره", "right": "که حق دارید اگر لطف بفرمایید بنده پیس به طور کامل اشتراک", "label": "YES", "label_source": "hand label", "p_yes": 0.8807970809769885}
|
| 18 |
+
{"id": "c5c8a83fd4a562ca", "call_id": "7232a844b7800c8cf93eb7abfb7bfc016741cd5be38fa054abcc483a033b08ae", "stratum": "entity", "crm_names": [], "left": "شد بعد همین پی که با من تماس گرفته میگه من به", "original": "همسایهها", "proposed": "همسایهت", "right": "تحویل دادم همسایهای که نه میشناسه نه ازش کد تحویل گرفته میگه", "label": "NO", "label_source": "hand label", "p_yes": 0.22270014468263327}
|
| 19 |
+
{"id": "a5efc8e8a8c834d9", "call_id": "2d8571531cd41718ea75ace7517fc1b9eff5a84fff1119836d24efbdb64229ca", "stratum": "entity", "crm_names": [], "left": "ولی یکیشو با تاخیر ثبت کردین درسته این سفارش یه دونه انگشتر", "original": "جوپینگ", "proposed": "ژوپینگ", "right": "بوده یه دونه زیرپوش رکابی بوده بعد بفرمایید کدشو لطفا بله هفتاد", "label": "YES", "label_source": "hand label", "p_yes": 0.7057850286668833}
|
| 20 |
+
{"id": "be48a37e8c46012d", "call_id": "3651dab2649115b19557d703494c0b6f73a5c0e5a01c308c02c60e44cb7bed6c", "stratum": "phonetic_sub", "crm_names": [], "left": "که تبدیل اومده بود بعد زمینهشو دی دیدم بسته رو ظاهر انتقال", "original": "دادم", "proposed": "دادن", "right": "به انبار چطور میتونم پیگیری کنم که دوباره بستهمو بگیرم محبت میفرمایید", "label": "YES", "label_source": "hand label", "p_yes": 0.13296424296956394}
|
| 21 |
+
{"id": "9050ef456cbfa3a5", "call_id": "73957c0fa254afcd9efde09dc4ce32ab5981fd1c057802bab30845d26bdb779c", "stratum": "entity", "crm_names": [], "left": "چطور میتونم کمکتون کنم خیلی ممنون سفارشتون چه چیزی بوده آقای اسلامی", "original": "پانسفری", "proposed": "ترانسفری", "right": "بلوتوثی بازه زمانی شم درسته برای هفتم شهریور بوده مجدد هشتم شهریور", "label": "YES", "label_source": "hand label", "p_yes": 0.8670357534616673}
|
| 22 |
+
{"id": "698cf45162ad1c62", "call_id": "1e8a52d7331d638aa4e8d219c0a337e116df2194f16cfa445ca9d33d91ec3abd", "stratum": "entity", "crm_names": [], "left": "این موضوع رو کمی برای مشتریانمون مخصوصا مشتریان پلاسمون کمی مرغوم به", "original": "صرف ترش", "proposed": "صرفهترش", "right": "کنیم اما در مورد هزینه ارسالی که از طرف فروشندهها رقم میخوره", "label": "YES", "label_source": "hand label", "p_yes": 0.7310585458420867}
|
| 23 |
+
{"id": "6567cb59ddda94de", "call_id": "0ae4b3e95627364e58c0ec0a454c4af1675f13a9c16b35a3443f440979515f2a", "stratum": "phonetic_sub", "crm_names": ["عباسی", "محمد"], "left": "این مشخص نیست چقدر زمان میبره باشه مرسی تشکر ممنونم مرسی تشکر", "original": "خدانگهدارموسیقی", "proposed": "خدانگهدار", "right": "", "label": "YES", "label_source": "hand label", "p_yes": 0.5621765067844595}
|
| 24 |
+
{"id": "ce8e6f30f061a3f9", "call_id": "425914f3e890126b36fb2f9577e6c0e1475748810efc543a1d5a344d1bf7b03e", "stratum": "other_sub", "crm_names": [], "left": "آزیزیده باشه چیزی بگین با عقیچوتر بیاد اینا آهنین چهجوری آزیز میبینن", "original": "من", "proposed": "میشه لطفا", "right": "الان پیگیری کنید به من اطلاع بدین آنلاین من پشت خط منتظر", "label": "YES", "label_source": "hand label", "p_yes": 0.43782350893182403}
|
| 25 |
+
{"id": "0f97fab8c50a4f69", "call_id": "5c47a64e0f878fb2d6e103b23425a5d41370a860e97b694d9fdc2e29e669c46f", "stratum": "entity", "crm_names": ["رضا", "سیاحی", "نوروزپور", "کیان"], "left": "دو تا دنبل بود و یه دونه دسته که ارسال نشد یه", "original": "دنبلش", "proposed": "دمبلش", "right": "ارسال شده برام بعد الان مونده یه دنبل و دسته بازی دو", "label": "YES", "label_source": "hand label", "p_yes": 0.7057850263061594}
|
| 26 |
+
{"id": "bcbb174d648a4aa5", "call_id": "348aa1e868fc5d43faa94a43d7f11f963f0ce99218c20d67d648b8e412916ee3", "stratum": "phonetic_sub", "crm_names": ["آسیه", "اکرم", "طوسی", "کارگر"], "left": "قهوهای برام آوردن آه من میخوام عوضش کنم اگه میشه شریفها رو", "original": "سپید بیارم", "proposed": "سفید بیارن", "right": "چون لازم دارم اصلا کار رو لازم دارم ولی الان چون اصلا", "label": "YES", "label_source": "hand label", "p_yes": 0.6791786922463603}
|
| 27 |
+
{"id": "b7d2db6fffa2dae7", "call_id": "1f4a33c4fbd066b30c7de0f9083226ffe52ed50321e97a431ee7fb42d6587c4d", "stratum": "phonetic_sub", "crm_names": [], "left": "نشاولا سامانه که عذر میخوام ازتون بابت تاخیر بنده همین الان ثبت", "original": "پیگیریشو", "proposed": "پیگیریش رو", "right": "میزنم زنده باشید که واریزی تا بعد از ظهر براتون انجام بشه", "label": "NO", "label_source": "hand label", "p_yes": 0.029312232473644905}
|
| 28 |
+
{"id": "11800b9e15ae3b61", "call_id": "632552271eebe5872efaacecdb96c559ba6182a1ec76803ec946af9747581d30", "stratum": "entity", "crm_names": ["اکبررعیت", "ریحانه", "مونا", "هاشمی"], "left": "سلام خانم", "original": "ماشینه خانم ماشینه", "proposed": "هاشمی خوش آمدید", "right": "ببخشید من این سفارشم در حال انجامه در اعضا ما میخوایم جای", "label": "YES", "label_source": "hand label", "p_yes": 0.7772998503931388}
|
| 29 |
+
{"id": "f86a44a59684df88", "call_id": "3f5d5e03a97c7016d886af8b7a39fd36fb3110df3b388bec5c000b5fb15630d7", "stratum": "other_sub", "crm_names": [], "left": "الان بودی دیجیپی سرکار گفت شما رو برای شانزده میلیون شما به", "original": "من", "proposed": "ما", "right": "معرفی کردن که این خطایه خب الان اینو ولش من از ا��ن", "label": "NO", "label_source": "hand label", "p_yes": 0.2450850112209964}
|
| 30 |
+
{"id": "3a2c656c59e9a778", "call_id": "6febe12263e7ba0d15628889f7eab2a11cc9ee422106c77d4df327809a5e6e0b", "stratum": "long", "crm_names": ["سهیلا", "طحانی", "کریمی", "کیانا"], "left": "روزه فقط منتظرم یه کاتر برای من بیارید دست شما درد نکنه", "original": "خب تا کی بهمون خبر میدین", "proposed": "کدوم یکیش رو فرمودید", "right": "بهمون توی ارسال سرکار خانم عذرخواهی میکنم الو سلام تا کی بهمون", "label": "NO", "label_source": "hand label", "p_yes": 0.3775406628360589}
|
| 31 |
+
{"id": "c8ed8c163fd0efbd", "call_id": "3b5441724451366117d7411f8803fda2ad6f6e0ec92c3a3375b3c68694b5cf81", "stratum": "phonetic_sub", "crm_names": ["الهام", "محدثه", "محمدی", "پنجه", "کوبی"], "left": "یک چهل و دو همین سفارشتون بود درسته یه چند لحظه کوتاه", "original": "امکانش هست", "proposed": "امکان نشست", "right": "پشت خط بنده", "label": "NO", "label_source": "hand label", "p_yes": 0.16451646221982344}
|
| 32 |
+
{"id": "eec8b0580596c770", "call_id": "1c8e8f4f28a247216b485746426470e525c0749872c88e1c33f96892996d580f", "stratum": "phonetic_sub", "crm_names": [], "left": "ساعت بعدش مبلغ عودت داده میشهدر غیر این صورت امکان حالا کنسلی", "original": "نیسته", "proposed": "نیستش", "right": "زمانی که ارسال کردن یه جور تحویل نگیرید اگر کالا رو نخواستید", "label": "YES", "label_source": "hand label", "p_yes": 0.09534945873634199}
|
| 33 |
+
{"id": "9786418cdd7c0061", "call_id": "316be6143ead8301ef96dac1a4fd8cffc36642eaee67b1f79e72de5083641213", "stratum": "phonetic_sub", "crm_names": [], "left": "هفتاد و دو ساعت بعد اینکه پولو ریکسید باید پرینتشو از بانک", "original": "بگیریدعکسشو", "proposed": "بگیرید عکسش رو", "right": "بفرستید ما این کارو انجام دادیم دیروز عکسشو هم ارسال کردم به", "label": "YES", "label_source": "hand label", "p_yes": 0.07585817884232454}
|
| 34 |
+
{"id": "9d61ce21e4ec5eb0", "call_id": "62b667c1f3db11629e8758f90399b9f7ea376e66160b933ef3e436a91861f7e4", "stratum": "other_sub", "crm_names": [], "left": "که بابت مابه تفاوت قیمت بهتون تا چند ساعت دیگه اساماس میاد", "original": "بعد", "proposed": "و", "right": "دیگه الان فکر کنم یه هفت هشتمین باره دارم الان تماس میگیرم", "label": "NO", "label_source": "hand label", "p_yes": 0.11920291761708243}
|
| 35 |
+
{"id": "57828737f1d60d5c", "call_id": "688f872a1c0c0992c271ac3c3ee5029d116803ed23105fac529645dce00451ba", "stratum": "other_sub", "crm_names": ["آقایی", "سلوی", "فخری", "محمدعلی"], "left": "هشت ساعت دیگه هفتاد و دو ساعت دو روز یا سه روز", "original": "که بشه", "proposed": "دیگه میشه", "right": "یک ماهه که این دیجیکالا پول منو هنوز واریز نکرده زنگ زدم", "label": "YES", "label_source": "hand label", "p_yes": 0.40733340735993506}
|
| 36 |
+
{"id": "d33288e84766e5cd", "call_id": "616d46d1f80e57f6d74dd821fd61555de17c56a1dd1f465b05078fde491d6343", "stratum": "entity", "crm_names": ["آیدا", "ابوالفضل", "حیدری", "شهبازیان", "کایدان"], "left": "قبلا تماس گرفتم امروز یه خانمه بهم گفت که برو تو سایت", "original": "ای این", "proposed": "ایرانجیاس اونجا آیامای", "right": "رو وارد کن با شمارهای که اونجا بهت میده دوباره تماس بگیر", "label": "YES", "label_source": "hand label", "p_yes": 0.4073334015815524}
|
| 37 |
+
{"id": "eb6242b0eea8db03", "call_id": "6aec0520a33cd564e33d13b011d00d5dbce30380f178ffaacd99ff83790ef70d", "stratum": "entity", "crm_names": [], "left": "که شما مراجعه کنید پیامش اومد مراجعه کنید اگر هم نیومد آقای", "original": "جمشیدی", "proposed": "جمشیری", "right": "داخل اون مکان حضوری یه پیام دیگه براتون میآد یه لینک داخلشه", "label": "NO", "label_source": "hand label", "p_yes": 0.14804718896359642}
|
| 38 |
+
{"id": "09a56eac5cd920d0", "call_id": "126c0f8a5e18fe890c12a57f13c450642134623b2463729b864241425bf886f3", "stratum": "entity", "crm_names": [], "left": "و نیم مشتری دارم خب کرم پودرم من تموم شده یعنی ته", "original": "وبات ما", "proposed": "موادم و", "right": "همه رو دیگه با براش استفاده کردم اینم وسیله کار", "label": "YES", "label_source": "hand label", "p_yes": 0.6791786899742113}
|
| 39 |
+
{"id": "6a8b4113aac0c7ee", "call_id": "44b7df48b8be976397fd39a6aefc9297bf781047e8edbe93cb3f827575d2043b", "stratum": "entity", "crm_names": ["خواه", "عذراء", "فتحی", "نیکوئی", "کوثر"], "left": "خب دیجیکالا این کارها بعیده نمیخواد اصلا سفارش کنه من لغو کنم", "original": "کشم مالیبک بزن", "proposed": "کش هم مانیبک بزنه", "right": "من یازده خوردهای شب ده و نیم بود صحبت کردم یه آقایی", "label": "YES", "label_source": "hand label", "p_yes": 0.37754065848213353}
|
| 40 |
+
{"id": "0b4c721400c3c106", "call_id": "44e97b29c53f9a1f2affaa444710df49a14ab58ef840ba3e4ffe38efb4622651", "stratum": "other_sub", "crm_names": [], "left": "باشین براتون پیگیریش رو ثبت کردم مرسی ممنون میشم در نظر سنجی", "original": "بکنید من", "proposed": "مکالمه هم", "right": "شرکت کنم وقتتون بخیر خدانگهدار", "label": "YES", "label_source": "hand label", "p_yes": 0.6791786907404345}
|
| 41 |
+
{"id": "ffb79f46020777b9", "call_id": "14efdeb2af63be9000a8360d9475a696cc867ce7837347d43731e4c00bc750cd", "stratum": "phonetic_sub", "crm_names": ["باقری", "تکتم", "معینی", "ناهید", "گزورعلیا"], "left": "تایید شده و مبلغش عودت شده خدمتتون شما دو تا سفارش داشتین", "original": "پول پست", "proposed": "کولهپست", "right": "فقط برای این سفارشتون که نبوده اگر برای این سف سفارش تنها", "label": "NO", "label_source": "hand label", "p_yes": 0.7310585502700596}
|
| 42 |
+
{"id": "5d2d254ac54bbeaf", "call_id": "23b36aabf3e4af71c2dc0010b91ffb40de10db5c6c23a77ed1ba17b7780147ab", "stratum": "phonetic_sub", "crm_names": [], "left": "اومده چندین باره اومده خب این یازده ده نه شب میاره خب", "original": "مسلم", "proposed": "مسلما", "right": "اداره تحتی رو دیگه با علم بینک اینجا اداره از یازده شب", "label": "YES", "label_source": "hand label", "p_yes": 0.10669059664755572}
|
| 43 |
+
{"id": "4e2398775fdd4656", "call_id": "3d2345170d4a1dd4f20b0423ed33f2b98684220f956d387ca2db07b010084a57", "stratum": "other_sub", "crm_names": [], "left": "حضورتون بابت موردی که وجود اومده جهت حالا موضوع شکایتی که فرمودین", "original": "من", "proposed": "برای", "right": "درخواست تماس از واحد شکایت براتون ثبت میکنم همکارمون از واحد شکایت", "label": "NO", "label_source": "hand label", "p_yes": 0.4073334095898663}
|
| 44 |
+
{"id": "fb4ceef89494ca68", "call_id": "1152073af47c95767bfc321f4892c99c69c76fe3457d3088c5a6526a2183da90", "stratum": "other_sub", "crm_names": [], "left": "سه تا سفارش دادم الان با مشکل خورده هر که با هر", "original": "روشنی", "proposed": "روشی که", "right": "میریم درست نمیشه باید چیکار کنیم گفتن از کشتیبانی باهاتون تماس گرفتیم", "label": "YES", "label_source": "hand label", "p_yes": 0.14804718520659846}
|
| 45 |
+
{"id": "121de2a8f3f4b5e1", "call_id": "383bd479ccb44a0b2ef8082d42cc04a5d53e9d7ee24ae0ced260f1ff1d796dd1", "stratum": "phonetic_sub", "crm_names": [], "left": "درخواست رسیدگی میشه یا دو سه ساعت آینده ولی قطعا به تمامی", "original": "درخواست", "proposed": "درخواستها", "right": "رسیدگی میشه تماس گرفته میشه خدمتتون و آدرس و بازه زمانی از", "label": "YES", "label_source": "hand label", "p_yes": 0.06754669241110224}
|
| 46 |
+
{"id": "da9b67e6ba27a716", "call_id": "68183cf653b28a8011b008b1ebd30098d75441d14a1c1181c097351045f6bde0", "stratum": "entity", "crm_names": [], "left": "سر هم زنگ زدم دیجیکالاچون شرکتهای دیگه که خدمات میدن از دیجیکالا", "original": "تازه کارت هرم", "proposed": "تازهکارتر هم", "right": "هستن گوش پیک رو میکشن میگن برو یا محصول رو تحویل بده", "label": "YES", "label_source": "hand label", "p_yes": 0.6224593305815925}
|
| 47 |
+
{"id": "9ae68ab5863ff667", "call_id": "45cdc49cd8b018d56032e0ae00a45aaaf998b10dddb94fa22c00dffe5b84fcfd", "stratum": "entity", "crm_names": [], "left": "چرا از دیشب تا حالا داره در حال بررسی الان من اون", "original": "غالب آقایی", "proposed": "قالبها هایی", "right": "که مرجوع کردم بعد هم ظرف یک دقیقه درخواست هم تایید شد", "label": "YES", "label_source": "hand label", "p_yes": 0.5621765010177999}
|
| 48 |
+
{"id": "22c9413544a965a3", "call_id": "492321acd54bdbbe0b5e0344278d291191a4c786c39285aa8bf561fd689d096a", "stratum": "other_sub", "crm_names": [], "left": "شخص دیگهای به جا شما حضور دارن باهاتون تماس گرفتن بهشون اعلام", "original": "میخواین به کالا", "proposed": "کنید تا", "right": "شخص دیگه تحویل بدن کالاتونو متاسفانه شماره مامور ارسال امکان پذیر نیست", "label": "YES", "label_source": "hand label", "p_yes": 0.7057850320687977}
|
| 49 |
+
{"id": "51e4a28311b1a305", "call_id": "376a9d0941cc82ac550d25d75fbee5ce24d3eae22b36302d697c0a9ae07fe7ab", "stratum": "other_sub", "crm_names": ["زاده", "سپیده", "شميل", "علی", "معين", "پور"], "left": "میشه امکانش هست بهم بگید که کدوم یکی از قضاها خدمتتون نرسیده", "original": "فی", "proposed": "فیتس", "right": "مولتیکالر بله چند لحظه صبوری بفرمایید جان دلم ام ام بفرمایید سرکار", "label": "YES", "label_source": "hand label", "p_yes": 0.13296424333379217}
|
| 50 |
+
{"id": "6b4f887e6221cd1a", "call_id": "53d49fce4aef900a662bceb1fadb3a1f5ced6c5359faa9766d9f19ef98a29b1a", "stratum": "long", "crm_names": [], "left": "دوباره اون صندلی بخرم دیدم سی تومنه بله بله بله چی میتونم", "original": "باید چه بله چه صندلشون", "proposed": "بخرم بعدش بعد این صندلی رو چون", "right": "یه دونه بیشتر نبود یهو بخرن که مثل مثل قبل نمیشه که", "label": "YES", "label_source": "hand label", "p_yes": 0.7981867852228951}
|
| 51 |
+
{"id": "c3a4ba9e27816738", "call_id": "395f6dea50fc6983dbfafaf043d24c93043f8955f2352c9961fa9eebde16eda2", "stratum": "entity", "crm_names": [], "left": "من پشت خط میمونم الان من یه بار چک کنم چون الان", "original": "دیدم", "proposed": "دیدمت", "right": "خب ببخشید خب من الان دارم چیزو میبینم رسید خرید میبینم فاکتور", "label": "NO", "label_source": "hand label", "p_yes": 0.1645164532005382}
|
| 52 |
+
{"id": "24c5a3ff1f223346", "call_id": "32ca3b36cc31df3719de4bfa886af92c75517d2a09464044eef59ae28e59e168", "stratum": "other_sub", "crm_names": [], "left": "دارم یعنی قابل هم اینکه نه جعبه کالا الان آقا خواستید دوباره", "original": "هم چکی", "proposed": "همون شکلی", "right": "بوده یعنی اصلا جعبهای نداشته به خدا الان دیگه دوباره چیزو خودش", "label": "YES", "label_source": "hand label", "p_yes": 0.7981867707049676}
|
| 53 |
+
{"id": "1c7e79919ecac506", "call_id": "5ab54e490b01b149b397555c0eab193f28c97389ece0bd0135fbc7458653f4cf", "stratum": "phonetic_sub", "crm_names": [], "left": "تو سیستم که سفارشتون تو راهه سفارشتون با کمی تاخیر تو راهه", "original": "آخرشم", "proposed": "آخرش هم", "right": "سفارش نیومده تا الان الان هم زنگ زدم میگم آقا چرا سفارش", "label": "NO", "label_source": "hand label", "p_yes": 0.09534945943402487}
|
| 54 |
+
{"id": "13228ec3b75a8dab", "call_id": "62b860f2ab42c41931fb9d1a2114161f4a7ddfa40c01495ee6e0309cea4bf59e", "stratum": "entity", "crm_names": [], "left": "لازم رو باهاتون انجام میدن به روی چشم به روی چشم خانم", "original": "رضای منش", "proposed": "رضویمنش", "right": "من درک میکنم که چقدر این مورد برای شما حساس هستش در", "label": "YES", "label_source": "hand label", "p_yes": 0.20181321998270516}
|
| 55 |
+
{"id": "0b50f03b708c47b2", "call_id": "4bd124ba9ed40dbb37c9b69e04a0a0e075e52aadfa6dc99e9b42d57f5376cb4f", "stratum": "other_sub", "crm_names": ["حمید", "فروغی"], "left": "حتما کارتون انجام میشه نگران نباشید بله سفارشتون رو هم چک بکنم", "original": "بعد", "proposed": "فقط", "right": "اینکه خروج از مرکز پردازش هستش که اون سفارشتون رو توی بازه", "label": "YES", "label_source": "hand label", "p_yes": 0.2942149776862473}
|
| 56 |
+
{"id": "b8579f8e67afe1f3", "call_id": "6b4c901c3797bca2148dc4cf660a0d3ac09196c0e0ce98a08712de4b222d93c5", "stratum": "entity", "crm_names": [], "left": "با سیم بوده هشتصد هزار تومن قیمتشهیعنی توی سفارشها بگردین یکی از", "original": "این هدستا", "proposed": "اون هدستها", "right": "هشتصد تومن قیمتشه هی صفر پنج همچین چیزی بله هدست لنوو هی", "label": "NO", "label_source": "hand label", "p_yes": 0.6513548521702845}
|
| 57 |
+
{"id": "3b4c6b2d2efb83fa", "call_id": "5196069b45d9f51dc4520a566f04f14b913d78f77ee89751f5b153211f675812", "stratum": "other_sub", "crm_names": [], "left": "چک میکنم چند لحظه اگه پیداش نکرده بودم بله لطف میکنید بخونید", "original": "صد و", "proposed": "ده تا", "right": "هفده بله بله سه بیست و هشت نود شصت و سه چند", "label": "NO", "label_source": "hand label", "p_yes": 0.2942149671868968}
|
| 58 |
+
{"id": "ed964f10c2c5d828", "call_id": "65f4114d76807c672d525a2eec5cf6110d7f8464be2e5a557e05801c10480644", "stratum": "entity", "crm_names": ["اسدی", "زهرا", "فرشته", "فیاض", "محرمی"], "left": "وقت بخیر", "original": "از ادیت", "proposed": "اسدی هستم", "right": "چطور میتونم راهنماییتون کنم بفرمایید لطفا بله خانم اهری بله ممنون الان", "label": "YES", "label_source": "hand label", "p_yes": 0.8354835402237746}
|
| 59 |
+
{"id": "22f228425759e8ad", "call_id": "3527c651686e8541aeceac1247b2e9b9df0927a543df3c2543c2d05229e0b86d", "stratum": "phonetic_sub", "crm_names": [], "left": "بررسی کنند بیست دقیقه بعدش تماس بگیرید بفرمایید که میخواید در واقع", "original": "عودت", "proposed": "عودات", "right": "وجه براتون انجام بشه متوجه هستم آقای شانگونی بله درسته اما خب", "label": "NO", "label_source": "hand label", "p_yes": 0.06754669352525676}
|
| 60 |
+
{"id": "94c75f3c0a2d8971", "call_id": "734e94aa0b7b059be3e659c109d57e1a7ed492361deec89ff83a8f6b1f8fdf7d", "stratum": "entity", "crm_names": [], "left": "دستتون درد نکرده این سواله دیگه این پست از نظر شما امروز", "original": "میآد", "proposed": "میادت", "right": "این کابلیهارا خب از نظر شما", "label": "NO", "label_source": "hand label", "p_yes": 0.40733339501364857}
|
| 61 |
+
{"id": "98f3eabc185bb694", "call_id": "6fb2a44632e6ef3f2bd90a307b1259f7b13ed74c7d9d4367b44cfe35740f43cb", "stratum": "entity", "crm_names": [], "left": "خسته نباشید میشه به واحد بررسی یا شکایت بسلم کنید آخه سفارشات", "original": "خیلی هست", "proposed": "خیلیان", "right": "که کنسل میشن میرسن عوض بسه نمیرسه یا اطلاعرسانی نمیکنن یا بسیار", "label": "NO", "label_source": "hand label", "p_yes": 0.16451645664759912}
|
| 62 |
+
{"id": "4234c8ce78c91209", "call_id": "203a1548f64f8816ca93fc08c5382b0e323077552584aaab707ec8518810fa46", "stratum": "other_sub", "crm_names": [], "left": "بخیر ظهراوی هستم چطور میتونم راهنماییتون کنم", "original": "بابت", "proposed": "سلام", "right": "ممنونم هنوز ثبت سفارشش رو انجام ندادین آقای حسن پیکینژاد درسته شاملیتون", "label": "YES", "label_source": "hand label", "p_yes": 0.08509903942020751}
|
| 63 |
+
{"id": "3df062f2cf68f4a9", "call_id": "5d8e5f7edb1fcf651bf2402c836fcbf2f369a9e2c4bf652ac7ef4f33990ae0d7", "stratum": "entity", "crm_names": [], "left": "بوده ولی اینا هر چیزا جدا جدا اومد من الان در مورد", "original": "بادی اطلاع شدم", "proposed": "بادیاسپلشه دارم", "right": "صحبت میکنم همین الان همین الان دریافت کردم ولی متاسفانه برای بار", "label": "YES", "label_source": "hand label", "p_yes": 0.7310585699310861}
|
| 64 |
+
{"id": "378d082bbaf80596", "call_id": "7456068cabb5547461ab8c100954f1fc3355da188ff3b27dbe339d80f47bef98", "stratum": "phonetic_sub", "crm_names": ["زهرا", "مقدم"], "left": "اعلام و نهایتا مبلغش واسهتون عودت میخورهمورد دیگهای هم باشه در خدمتتون", "original": "هستم", "proposed": "هستیم", "right": "الان من براتون بازه رو تغییر دادم آدرسم تغییر دادم پیامک تاییدش", "label": "NO", "label_source": "hand label", "p_yes": 0.09534946120112255}
|
| 65 |
+
{"id": "668d4d7dd84fbe53", "call_id": "327900fa1c2329c87119570b382702d9917ed93afaabc81ce9a2cab747f3ea93", "stratum": "phonetic_sub", "crm_names": ["آرمات", "داریوش", "فاطمه", "همت", "پور"], "left": "این دوستمون که محصول سال سلام خوبی این اومدن دو تا از", "original": "کالا", "proposed": "کالاها", "right": "رو آورده بودن یکیشو نیاورده بودن و ما اجور شدیم برگردونیم دو", "label": "YES", "label_source": "hand label", "p_yes": 0.053403330713777154}
|
| 66 |
+
{"id": "21836997c55f14c4", "call_id": "1ca703abb847bfe9e5627748c0eeba080626d186475f8c5a41aeadb983958c41", "stratum": "phonetic_sub", "crm_names": [], "left": "ارسال میشه نگران نباشین بهتون اطلاعرسانیش انجام بشه زنده باشین نظرسنجی تماس", "original": "بنده", "proposed": "بندر", "right": "رو ممنون میشم شرکت فرمایید روز خوبید", "label": "NO", "label_source": "hand label", "p_yes": 0.06008664541758885}
|
| 67 |
+
{"id": "c223792c693be471", "call_id": "32608f2f93530548653aa69f2414c5d23e208ac628844a33ae68f451f8d93265", "stratum": "entity", "crm_names": [], "left": "هم تماس گرفتم خدمتتون من با کارت هدیه خرید کردم از طریق", "original": "کمکسایت جهیزیه میخواستم", "proposed": "کمیکبای جهزیه خواستم", "right": "ببینم چطوری میتونم لغو داشته باشم نه نه نه صفر نهصد و", "label": "NO", "label_source": "hand label", "p_yes": 0.37754067370267064}
|
| 68 |
+
{"id": "4dab1fe108f4ee5b", "call_id": "4ff85d0a83c00b6f64660a0217923e1173c382c61f2a1b3304fbd55bdbc2a8d3", "stratum": "phonetic_sub", "crm_names": ["حسینی", "داوری", "سیده", "فاطمه", "مهتاب", "پورمقدم"], "left": "از طریق پیامک بهتون اطلاعرسانیش انجام میشه خواهش میکنم ممنون میشم به", "original": "نظرسنجیم", "proposed": "نظرسنجیمون", "right": "پاسخ بدید روزتون بخیر خداحافظ", "label": "NO", "label_source": "hand label", "p_yes": 0.1192029239097348}
|
| 69 |
+
{"id": "f993a1b2c28e01ad", "call_id": "1d9faf79cf7346142dfcdaa586ad0bb85bb00b10b1da24dd93d29e2f5dbec616", "stratum": "entity", "crm_names": ["ابادی", "حاجی", "خرمی", "مهتاب", "مهدی"], "left": "میخواین یعنی لغوش براتون انجام شد ماگ هستش برمیدین من نگاه کنم", "original": "مادی", "proposed": "ماگ", "right": "رو براتون یه ماگ سفری بوده من دارم نگاه میکنم بله این", "label": "YES", "label_source": "hand label", "p_yes": 0.7310585529722567}
|
| 70 |
+
{"id": "db266bd810b2b63f", "call_id": "6ed6875cadc80b467a7728dd046b17f5dc1e3a460ba28ec66603a4c421ef62ff", "stratum": "phonetic_sub", "crm_names": [], "left": "میشه بابت تاخیرش عذرخواهی میکنم بله بله کالا ارسال میشه خواهش میکنم", "original": "سوال", "proposed": "سؤال", "right": "دیگهای هست کمکتون کنم ممنون میشم نظرسنجی کسایی خداحافظ", "label": "NO", "label_source": "hand label", "p_yes": 0.025957356076576288}
|
| 71 |
+
{"id": "47d0746afd4387b9", "call_id": "1d907659f4aecd82a63a67819454d33b4c5277ee87f03f7c0cdd8f5580be9577", "stratum": "phonetic_sub", "crm_names": [], "left": "هزینه ارسال خیلی کم بود و اصلا من فکر میکنم به این", "original": "دلیل", "proposed": "دلیله", "right": "که نفرستادن برای من این کالا روحالا میخوام بدونم مثلا اونو برای", "label": "NO", "label_source": "hand label", "p_yes": 0.0850990444191706}
|
| 72 |
+
{"id": "7c5b96b54f1c0a4f", "call_id": "3b0f060b520e9dccadb7e79bda08ecbc5ef3f57d75ccc62150c12105c0acf90e", "stratum": "other_sub", "crm_names": [], "left": "ما در حضور پنج دقیقه پیش هم صحبت کردیم نمیدونم تا چه", "original": "بفرسته", "proposed": "اپراتوری بود", "right": "یه خرید حدودا یک میلیون و سیصد و شش هزار تومنی ما", "label": "YES", "label_source": "hand label", "p_yes": 0.7549149966191566}
|
| 73 |
+
{"id": "8470218a56683097", "call_id": "010d00c637c3972981df4b5ab2d87f3253d69d9b19d02041f7530336bba066ab", "stratum": "entity", "crm_names": [], "left": "میخریم حالا جدا از هزینه رایگان ارسالش باید ببینیم بیشتر به خاطر", "original": "دقتی", "proposed": "دقتیه", "right": "که بیشتر بشه چون قبل از این من سه بار تقریبا داستان", "label": "NO", "label_source": "hand label", "p_yes": 0.11920292097541764}
|
| 74 |
+
{"id": "3c803d8c1a64d3ce", "call_id": "2778f3fab1e70096d4c326d76d074453dc28fb4e52cf71c9dafa17544dd28e59", "stratum": "other_sub", "crm_names": [], "left": "یا خارج از تایم خدمتتون تماس بگیرن سلامت باشید روزتون بخیر ممنون", "original": "از دسترسی", "proposed": "که نظرسنجی", "right": "کنید تماس شرکت کنید خدانگهدار شما باشه", "label": "YES", "label_source": "hand label", "p_yes": 0.7549149912586075}
|
| 75 |
+
{"id": "0e99bc7b3c1a6e8f", "call_id": "4df2a7995fc6f96cba79c996da45f2c1b9afdb74d1ee5cb7109cbffeeed2c2f1", "stratum": "other_sub", "crm_names": [], "left": "دقیق هستم چطور میتونم راهنماییتون کنم سلام بفرمایید عذرخواهی میکنم جناب", "original": "غیر", "proposed": "آقای", "right": "نظری میفرمایید که چند مرتبه این موضوع نسبت به لغو سفارش براتون", "label": "YES", "label_source": "hand label", "p_yes": 0.7549149867041595}
|
| 76 |
+
{"id": "868c5fab5c99a1a1", "call_id": "4ea4ffb3beb602178ebf1d06f6b2e29d6c7de8fc22704448d3e4b88d1513f0db", "stratum": "phonetic_sub", "crm_names": [], "left": "ارجاع میده واحد مربوطه براتون آقای صدقیان زنده باشین نظر سنجی تماس", "original": "بنده رو", "proposed": "بندر", "right": "ممنون میشم شرکت کنید روزتون بخیر", "label": "NO", "label_source": "hand label", "p_yes": 0.04742587038475011}
|
| 77 |
+
{"id": "2e7c71c9ae6eca06", "call_id": "1d92ceee8098b3cb3170a49c3e6b8919ddd13763fb8d6fccc5c2f5e06db426fd", "stratum": "phonetic_sub", "crm_names": [], "left": "خدمتتون که بابت موردی که میفرمایید اگر همچنان در حال بررسی هست", "original": "درخواستتونو", "proposed": "درخواستتون", "right": "نهایتا تا فردا لطف ک��ید صبوری بفرمایید اگر تماسی باهاتون برقرار نشد", "label": "NO", "label_source": "hand label", "p_yes": 0.060086649933198344}
|
| 78 |
+
{"id": "5f364b2724c245a8", "call_id": "1017520b3ca67c4632b885f9444f69d1b44bbb1f236ac81a52edfa957b769c36", "stratum": "other_sub", "crm_names": [], "left": "تخفیف قبلا استفاده شده کد رو ارسال کن چجوری ارسال کنم اون", "original": "مورد", "proposed": "ماده", "right": "کده که میاد میزنم روشنه نه کد چیز نمیشه کد برام پیامک", "label": "NO", "label_source": "hand label", "p_yes": 0.5312093708138155}
|
| 79 |
+
{"id": "6199ead61453d035", "call_id": "46d4615b14b54a9a97a239bb0ff371f57b2b10b0912facc4399814965d2de4ac", "stratum": "entity", "crm_names": [], "left": "دو ماهه یه عینک", "original": "چنزلی", "proposed": "چندزلی", "right": "سفارش دادم یکو خوردهای زدم به حسابشون برای اینکه برام ارسال نشد", "label": "NO", "label_source": "hand label", "p_yes": 0.43782350135935577}
|
| 80 |
+
{"id": "a51b1e0aad653977", "call_id": "10b2b14d9ed8bd8dc153b9eb7e7bc76f1e22cd61b231b461eada1a6fc04257c8", "stratum": "phonetic_sub", "crm_names": [], "left": "وقت بخیر", "original": "شهربازیان", "proposed": "شهبازیان", "right": "هستم چطور میتونم راهنماییتون کنم زنده باشید متشکرم بله چک میکنم براتون", "label": "YES", "label_source": "hand label", "p_yes": 0.2689414470924797}
|
| 81 |
+
{"id": "d4852124735bb65f", "call_id": "485a8b61355d127eb85936b9cb06a7b61f1d6811136e8fa2ea9d258a86e69bce", "stratum": "phonetic_sub", "crm_names": ["محسن", "کوپشی"], "left": "قطع شد نمیفهمم این باید امروز به دست من برسه من دیگه", "original": "نمیدونم", "proposed": "نمیتونم", "right": "پنج روز به خاطر پنج کیلو برنج مرخصی بگیرم دوست عزیز که", "label": "YES", "label_source": "hand label", "p_yes": 0.20181322236974344}
|
| 82 |
+
{"id": "380bba982a8d7e76", "call_id": "6334c9593cdf5bac91dde2b7ec825dd345788db15574d32b06a1fa2ac77740c3", "stratum": "other_sub", "crm_names": [], "left": "که تاریخ بعد شماره یه نفر به من زنگ زد گفت آقا", "original": "عادتت کجا", "proposed": "آدرست کجاست", "right": "گفتم فردی ست عادت دادم گفت من فاضیه که اندیشم گفتم آقا", "label": "YES", "label_source": "hand label", "p_yes": 0.8933094056944728}
|
| 83 |
+
{"id": "6c5ebb923588da89", "call_id": "0a5380f74a26f6e3caca55e7052c2ef6c135860e75b397e2f653fa7ce2c5663a", "stratum": "phonetic_sub", "crm_names": [], "left": "تقریبا دیروز لغو کردم بعد این وجهش برام واریز نشده میخواستم ببینم", "original": "چجوریه", "proposed": "چهجوریه", "right": "پیگیری کنم ببینم کی قراره واریز بشه دیروز یا پریروز لغو کردم", "label": "NO", "label_source": "hand label", "p_yes": 0.04742587440546322}
|
| 84 |
+
{"id": "c766cadc8ef5912c", "call_id": "2ce9c6ffcea9a4a834126e9bd0bf9253150f3548a228489c8101432791b6c8f2", "stratum": "phonetic_sub", "crm_names": ["خزایی", "فاطمه", "محمودخانی", "هانیه"], "left": "که بتونین ثبت مرجوعی انجام بدین بله بله آخرین وضعیتش در واقع", "original": "سنتو", "proposed": "سند تو", "right": "پسته یعنی تحویل مامور پستی شده و هنوز مامور پستی تحویل کامل", "label": "YES", "label_source": "hand label", "p_yes": 0.24508501531567162}
|
| 85 |
+
{"id": "3e52b593617751a8", "call_id": "6bc6a5664f5bb5cc088ef92fd5baff2e6951587139176b397fd2858ca24e9a11", "stratum": "entity", "crm_names": [], "left": "لو بذار پولشو خیلی خب پولشو بردارید در کانت منو باز کنید", "original": "خیلی چی مسخره بازیها", "proposed": "آخه این چه مسخرهبازیهاست", "right": "چیه اصلا بعد صبح ساعت هشت تماس گرفتم همه اینا رو گفتن", "label": "YES", "label_source": "hand label", "p_yes": 0.8175744707778093}
|
| 86 |
+
{"id": "a911a4a7f1e6c7a9", "call_id": "663a2b18b82243bce1d613320771876610b7e4f7be275f6b1bd2d9c62e5d8d9b", "stratum": "other_sub", "crm_names": ["آریا", "تمسکنی", "زاهدی"], "left": "همچین چیزی نیستش اونجا که میخوام تعریفش کنم بله بله اگر زحمت", "original": "کشید", "proposed": "میکشی خودت", "right": "من الان بار فکر کنم بار دهم باشه دارم تماس میگیرم پیام", "label": "YES", "label_source": "hand label", "p_yes": 0.13296424790406106}
|
| 87 |
+
{"id": "5b3c02900a7605e0", "call_id": "61b17e3a5fd9d653cf3acf092bd0da32e70c882b383e75ce67c53ad5bd8892f9", "stratum": "phonetic_sub", "crm_names": [], "left": "سفارش خواهش میکنم الان این مورد که دوازدهم تقدیمتون نیسته ولی اگه", "original": "سوال", "proposed": "سؤال", "right": "دیگهای در خدمتتون هستم اگر سفارش دیگهای مد نظرتون هست کدشو بفرمایید", "label": "NO", "label_source": "hand label", "p_yes": 0.04208773095275436}
|
| 88 |
+
{"id": "fd25fa55eb0e25a0", "call_id": "6b0b106524e06a444a3da49c139eddf64cce02be1458fcbbd6d0d943ba6cabf7", "stratum": "phonetic_sub", "crm_names": [], "left": "شماره سه شماره چهار یعنی دویی هم ارسال شده اون زمان به", "original": "احواز", "proposed": "اهواز", "right": "اصلا برای چه کالاییه من مشکل من دو تا سفارش اومد که", "label": "YES", "label_source": "hand label", "p_yes": 0.5312093670239941}
|
| 89 |
+
{"id": "a07be1fb2564a90b", "call_id": "0a286b4209f9974b5340f647dbfd072c045e236f49c7aee26730aff555033c4e", "stratum": "phonetic_sub", "crm_names": [], "left": "وقتتون بخیر چند لحظه منتظر باشید سفارشتون رو پیگیری کنم جناب آقای", "original": "مسعوده", "proposed": "مسعود", "right": "دالایی درسته سفارش برای گنجه ارسالیاش انجام شده اما هنوز تحویل گنجه", "label": "YES", "label_source": "hand label", "p_yes": 0.16451645301506868}
|
| 90 |
+
{"id": "488bef693c19fadd", "call_id": "18a8722136f139e0b11fa8402a0d5451f9e9673cefc910d29659e35e54cb1c9d", "stratum": "phonetic_sub", "crm_names": [], "left": "شکایت بشه باشه در حق حقوقی میرم شکایت میکنم چون واقعا خیلی", "original": "مسخره است", "proposed": "مسخرهست", "right": "یعنی اگه امروز دیگه تا آخر وقت دیگه روز آخره دیگه من", "label": "NO", "label_source": "hand label", "p_yes": 0.06754669544354523}
|
| 91 |
+
{"id": "e9cbb37ea2fb450a", "call_id": "110f3585c287f25466c0ace039078e3ff00893afa77d7238891f0b2cab410c7f", "stratum": "phonetic_sub", "crm_names": ["بوسجین", "نادیا", "نورانی", "پریسا", "کبودانی"], "left": "گفتم که چ نه نه نه نه ببینید دستهبندی خود دیجیکالا کامل", "original": "پلمپ", "proposed": "پلمب", "right": "بود خب خود کارتن بعد داخلش یه پلاستیک بود اون پلاستیکه هم", "label": "YES", "label_source": "hand label", "p_yes": 0.26894143324671577}
|
| 92 |
+
{"id": "4d84471775b145e6", "call_id": "340e5d9d8cfc834c07314eeab9943b5f9305ae2c2ce0dd1112b7cf2ac943d64a", "stratum": "phonetic_sub", "crm_names": ["آقبلاغ", "اخلاقی", "اقبال", "علی", "نسترن", "هاشم", "پور"], "left": "جناب امینی سفارشتون قفط هما شیرالات بوده که یه موردش رو دریافت", "original": "نکردید", "proposed": "نکردیم", "right": "جناب امینی بابت این مورد بنده حق میدم نارضی باشید درخواست پیگیری", "label": "NO", "label_source": "hand label", "p_yes": 0.09534945884249486}
|
| 93 |
+
{"id": "1252717c36ff426b", "call_id": "5db5c5ad848bd941c06a21b65f066996b90ae7a9c0a8a37aae03f2edd61c26c2", "stratum": "phonetic_sub", "crm_names": [], "left": "رایالیهای هستش من منتظر بدم من یعنی من کار خودمو میکنم ببین", "original": "ارزش این", "proposed": "ارزشی", "right": "نداره این پولش ارزش نداره از دلیل هیسیتی برای من خیلی بده", "label": "YES", "label_source": "hand label", "p_yes": 0.1645164469703713}
|
| 94 |
+
{"id": "48e7524914c8c36d", "call_id": "67a07314e009083fd536a603267b722a886ed3fc3d214cdad060852fceed9802", "stratum": "other_sub", "crm_names": [], "left": "تشکر ازتون نه نه فقط همین که گفتم ببینم چرا تاخیر داره", "original": "اونو", "proposed": "و", "right": "نرسیده برای اونم ببینم اگه میخواست رز بشه برام بفرستین گلبهار تغییر", "label": "NO", "label_source": "hand label", "p_yes": 0.1645164633920197}
|
| 95 |
+
{"id": "2b2519d1b6df22fd", "call_id": "52e611b4f7fd94247b8ba8b381446ca29bcaea32c59745c9c3ca3b659cd142e5", "stratum": "entity", "crm_names": [], "left": "خانم شما بخیر خسته نباشید من", "original": "روی شم داره چی", "proposed": "درویشم یه ارسال", "right": "داشتم به شماره سیصد و هفتاد و هفت شصت و شش هفتاد", "label": "YES", "label_source": "hand label", "p_yes": 0.9324533073961285}
|
| 96 |
+
{"id": "ca4ac695b3677eb4", "call_id": "41f05865698927f91648226c82eab76c0211fa917eb3d6178a90f3f6b6a978e2", "stratum": "phonetic_sub", "crm_names": [], "left": "آقای بهبهران نهار بهروزرسانی هستم ولی برای اینکه من راهنماییتون کنم بعدا", "original": "بخوایم", "proposed": "بخواین", "right": "پیگیری ثبت کنیم اه سفارشها وقتی که از سمت دیجیکالا لغو میشه", "label": "NO", "label_source": "hand label", "p_yes": 0.18242553346464097}
|
| 97 |
+
{"id": "1641dfb6750655c1", "call_id": "71bb88eceaf01eb7e20b2cccc29f1635c1f5fe1a016fc8d6c21f03c2848730f7", "stratum": "other_sub", "crm_names": ["بهاره", "حسن", "سادات", "صادقی", "مطیعی", "مهر"], "left": "تو دیجیکالای من الان نگاه کردم زده امکان مرجوعی وجود نداره آها", "original": "بچه رو", "proposed": "پس چرا", "right": "تو سیستم میزنه امکان مرجوعی وجود نداره آه اشکال نداره بله چون", "label": "YES", "label_source": "hand label", "p_yes": 0.7310585595368002}
|
| 98 |
+
{"id": "7a3e5518b5d677f3", "call_id": "19df63b7b0986197496cd90d3d0393fa9a7dd30e004a721a0f7d11163a88eb3e", "stratum": "entity", "crm_names": ["حانیه", "قربانی", "مریم", "نوری"], "left": "هزینه مرسولهها که به حسابم برگشت میخوره معمولا تازه الان لغو آها", "original": "حداکثر", "proposed": "حداک", "right": "متشکرم از شما مرسی خسته نباشید نه مرسی متشکرم خسته نباشید بله", "label": "NO", "label_source": "hand label", "p_yes": 0.025957356644309044}
|
| 99 |
+
{"id": "5906a8cb68dadbfd", "call_id": "035407afaeedc07385c085fd4404057c73a0b4d2ff263efb5d81ec3d562e2816", "stratum": "entity", "crm_names": [], "left": "که دیروز میاد به دستم میرسه منم دیروز خونه نبودم و با", "original": "همکارتون", "proposed": "همکارتا", "right": "هماهنگ کردم که نیاد بعد میخواستم ببینم که امروز هم برای من", "label": "NO", "label_source": "hand label", "p_yes": 0.060086646357115164}
|
verifier-qwen3.5-2b/chat_template.jinja
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{%- set image_count = namespace(value=0) %}
|
| 2 |
+
{%- set video_count = namespace(value=0) %}
|
| 3 |
+
{%- macro render_content(content, do_vision_count, is_system_content=false) %}
|
| 4 |
+
{%- if content is string %}
|
| 5 |
+
{{- content }}
|
| 6 |
+
{%- elif content is iterable and content is not mapping %}
|
| 7 |
+
{%- for item in content %}
|
| 8 |
+
{%- if 'image' in item or 'image_url' in item or item.type == 'image' %}
|
| 9 |
+
{%- if is_system_content %}
|
| 10 |
+
{{- raise_exception('System message cannot contain images.') }}
|
| 11 |
+
{%- endif %}
|
| 12 |
+
{%- if do_vision_count %}
|
| 13 |
+
{%- set image_count.value = image_count.value + 1 %}
|
| 14 |
+
{%- endif %}
|
| 15 |
+
{%- if add_vision_id %}
|
| 16 |
+
{{- 'Picture ' ~ image_count.value ~ ': ' }}
|
| 17 |
+
{%- endif %}
|
| 18 |
+
{{- '<|vision_start|><|image_pad|><|vision_end|>' }}
|
| 19 |
+
{%- elif 'video' in item or item.type == 'video' %}
|
| 20 |
+
{%- if is_system_content %}
|
| 21 |
+
{{- raise_exception('System message cannot contain videos.') }}
|
| 22 |
+
{%- endif %}
|
| 23 |
+
{%- if do_vision_count %}
|
| 24 |
+
{%- set video_count.value = video_count.value + 1 %}
|
| 25 |
+
{%- endif %}
|
| 26 |
+
{%- if add_vision_id %}
|
| 27 |
+
{{- 'Video ' ~ video_count.value ~ ': ' }}
|
| 28 |
+
{%- endif %}
|
| 29 |
+
{{- '<|vision_start|><|video_pad|><|vision_end|>' }}
|
| 30 |
+
{%- elif 'text' in item %}
|
| 31 |
+
{{- item.text }}
|
| 32 |
+
{%- else %}
|
| 33 |
+
{{- raise_exception('Unexpected item type in content.') }}
|
| 34 |
+
{%- endif %}
|
| 35 |
+
{%- endfor %}
|
| 36 |
+
{%- elif content is none or content is undefined %}
|
| 37 |
+
{{- '' }}
|
| 38 |
+
{%- else %}
|
| 39 |
+
{{- raise_exception('Unexpected content type.') }}
|
| 40 |
+
{%- endif %}
|
| 41 |
+
{%- endmacro %}
|
| 42 |
+
{%- if not messages %}
|
| 43 |
+
{{- raise_exception('No messages provided.') }}
|
| 44 |
+
{%- endif %}
|
| 45 |
+
{%- if tools and tools is iterable and tools is not mapping %}
|
| 46 |
+
{{- '<|im_start|>system\n' }}
|
| 47 |
+
{{- "# Tools\n\nYou have access to the following functions:\n\n<tools>" }}
|
| 48 |
+
{%- for tool in tools %}
|
| 49 |
+
{{- "\n" }}
|
| 50 |
+
{{- tool | tojson }}
|
| 51 |
+
{%- endfor %}
|
| 52 |
+
{{- "\n</tools>" }}
|
| 53 |
+
{{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
|
| 54 |
+
{%- if messages[0].role == 'system' %}
|
| 55 |
+
{%- set content = render_content(messages[0].content, false, true)|trim %}
|
| 56 |
+
{%- if content %}
|
| 57 |
+
{{- '\n\n' + content }}
|
| 58 |
+
{%- endif %}
|
| 59 |
+
{%- endif %}
|
| 60 |
+
{{- '<|im_end|>\n' }}
|
| 61 |
+
{%- else %}
|
| 62 |
+
{%- if messages[0].role == 'system' %}
|
| 63 |
+
{%- set content = render_content(messages[0].content, false, true)|trim %}
|
| 64 |
+
{{- '<|im_start|>system\n' + content + '<|im_end|>\n' }}
|
| 65 |
+
{%- endif %}
|
| 66 |
+
{%- endif %}
|
| 67 |
+
{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
|
| 68 |
+
{%- for message in messages[::-1] %}
|
| 69 |
+
{%- set index = (messages|length - 1) - loop.index0 %}
|
| 70 |
+
{%- if ns.multi_step_tool and message.role == "user" %}
|
| 71 |
+
{%- set content = render_content(message.content, false)|trim %}
|
| 72 |
+
{%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
|
| 73 |
+
{%- set ns.multi_step_tool = false %}
|
| 74 |
+
{%- set ns.last_query_index = index %}
|
| 75 |
+
{%- endif %}
|
| 76 |
+
{%- endif %}
|
| 77 |
+
{%- endfor %}
|
| 78 |
+
{%- if ns.multi_step_tool %}
|
| 79 |
+
{{- raise_exception('No user query found in messages.') }}
|
| 80 |
+
{%- endif %}
|
| 81 |
+
{%- for message in messages %}
|
| 82 |
+
{%- set content = render_content(message.content, true)|trim %}
|
| 83 |
+
{%- if message.role == "system" %}
|
| 84 |
+
{%- if not loop.first %}
|
| 85 |
+
{{- raise_exception('System message must be at the beginning.') }}
|
| 86 |
+
{%- endif %}
|
| 87 |
+
{%- elif message.role == "user" %}
|
| 88 |
+
{{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
|
| 89 |
+
{%- elif message.role == "assistant" %}
|
| 90 |
+
{%- set reasoning_content = '' %}
|
| 91 |
+
{%- if message.reasoning_content is string %}
|
| 92 |
+
{%- set reasoning_content = message.reasoning_content %}
|
| 93 |
+
{%- else %}
|
| 94 |
+
{%- if '</think>' in content %}
|
| 95 |
+
{%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
|
| 96 |
+
{%- set content = content.split('</think>')[-1].lstrip('\n') %}
|
| 97 |
+
{%- endif %}
|
| 98 |
+
{%- endif %}
|
| 99 |
+
{%- set reasoning_content = reasoning_content|trim %}
|
| 100 |
+
{%- if loop.index0 > ns.last_query_index %}
|
| 101 |
+
{{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content + '\n</think>\n\n' + content }}
|
| 102 |
+
{%- else %}
|
| 103 |
+
{{- '<|im_start|>' + message.role + '\n' + content }}
|
| 104 |
+
{%- endif %}
|
| 105 |
+
{%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}
|
| 106 |
+
{%- for tool_call in message.tool_calls %}
|
| 107 |
+
{%- if tool_call.function is defined %}
|
| 108 |
+
{%- set tool_call = tool_call.function %}
|
| 109 |
+
{%- endif %}
|
| 110 |
+
{%- if loop.first %}
|
| 111 |
+
{%- if content|trim %}
|
| 112 |
+
{{- '\n\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
|
| 113 |
+
{%- else %}
|
| 114 |
+
{{- '<tool_call>\n<function=' + tool_call.name + '>\n' }}
|
| 115 |
+
{%- endif %}
|
| 116 |
+
{%- else %}
|
| 117 |
+
{{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
|
| 118 |
+
{%- endif %}
|
| 119 |
+
{%- if tool_call.arguments is defined %}
|
| 120 |
+
{%- for args_name, args_value in tool_call.arguments|items %}
|
| 121 |
+
{{- '<parameter=' + args_name + '>\n' }}
|
| 122 |
+
{%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
|
| 123 |
+
{{- args_value }}
|
| 124 |
+
{{- '\n</parameter>\n' }}
|
| 125 |
+
{%- endfor %}
|
| 126 |
+
{%- endif %}
|
| 127 |
+
{{- '</function>\n</tool_call>' }}
|
| 128 |
+
{%- endfor %}
|
| 129 |
+
{%- endif %}
|
| 130 |
+
{{- '<|im_end|>\n' }}
|
| 131 |
+
{%- elif message.role == "tool" %}
|
| 132 |
+
{%- if loop.previtem and loop.previtem.role != "tool" %}
|
| 133 |
+
{{- '<|im_start|>user' }}
|
| 134 |
+
{%- endif %}
|
| 135 |
+
{{- '\n<tool_response>\n' }}
|
| 136 |
+
{{- content }}
|
| 137 |
+
{{- '\n</tool_response>' }}
|
| 138 |
+
{%- if not loop.last and loop.nextitem.role != "tool" %}
|
| 139 |
+
{{- '<|im_end|>\n' }}
|
| 140 |
+
{%- elif loop.last %}
|
| 141 |
+
{{- '<|im_end|>\n' }}
|
| 142 |
+
{%- endif %}
|
| 143 |
+
{%- else %}
|
| 144 |
+
{{- raise_exception('Unexpected message role.') }}
|
| 145 |
+
{%- endif %}
|
| 146 |
+
{%- endfor %}
|
| 147 |
+
{%- if add_generation_prompt %}
|
| 148 |
+
{{- '<|im_start|>assistant\n' }}
|
| 149 |
+
{%- if enable_thinking is defined and enable_thinking is true %}
|
| 150 |
+
{{- '<think>\n' }}
|
| 151 |
+
{%- else %}
|
| 152 |
+
{{- '<think>\n\n</think>\n\n' }}
|
| 153 |
+
{%- endif %}
|
| 154 |
+
{%- endif %}
|
verifier-qwen3.5-2b/tb/events.out.tfevents.1790667609.02b48645612f.2184.0
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7bc2eaac2c343d7bd91daf1bdf9765ca53dc9753fbe76549cb353c91070c13ae
|
| 3 |
+
size 21219
|
verifier-qwen3.5-2b/test_eval/baselines.json
ADDED
|
@@ -0,0 +1,227 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"whisper": {
|
| 3 |
+
"n_examples": 1978,
|
| 4 |
+
"wer": 0.19286111707900203,
|
| 5 |
+
"wer_zwnj_normalized": 0.18077200636819868,
|
| 6 |
+
"exact_match": 0.006066734074823054,
|
| 7 |
+
"hallucination_rate": 0.0,
|
| 8 |
+
"rows_changed": 0.0,
|
| 9 |
+
"fix_rate": 0.0,
|
| 10 |
+
"preservation_rate": 1.0,
|
| 11 |
+
"targeted_score": 0.5,
|
| 12 |
+
"fix_rate_lenient": 0.0,
|
| 13 |
+
"preservation_rate_lenient": 1.0,
|
| 14 |
+
"targeted_score_lenient": 0.5,
|
| 15 |
+
"fixed_lenient": 0,
|
| 16 |
+
"broken_lenient": 0,
|
| 17 |
+
"net_fixed_lenient": 0,
|
| 18 |
+
"broken_per_100_correct": 0.0,
|
| 19 |
+
"fix_rate_recoverable": 0.0,
|
| 20 |
+
"unrecoverable_targets": 966,
|
| 21 |
+
"entity_fix_rate": 0.0,
|
| 22 |
+
"entity_keep_rate": 1.0,
|
| 23 |
+
"entity_spans_whisper_wrong": 57,
|
| 24 |
+
"entity_spans_whisper_right": 217,
|
| 25 |
+
"accepted": 0
|
| 26 |
+
},
|
| 27 |
+
"rules only": {
|
| 28 |
+
"n_examples": 1978,
|
| 29 |
+
"wer": 0.18624502140177493,
|
| 30 |
+
"wer_zwnj_normalized": 0.17446440474419783,
|
| 31 |
+
"exact_match": 0.006572295247724975,
|
| 32 |
+
"hallucination_rate": 0.0,
|
| 33 |
+
"rows_changed": 0.6875631951466128,
|
| 34 |
+
"fix_rate": 0.05559820155718829,
|
| 35 |
+
"preservation_rate": 0.9996089846635096,
|
| 36 |
+
"targeted_score": 0.5276035931103489,
|
| 37 |
+
"fix_rate_lenient": 0.05892479464856623,
|
| 38 |
+
"preservation_rate_lenient": 0.9995903045402917,
|
| 39 |
+
"targeted_score_lenient": 0.5292575495944289,
|
| 40 |
+
"fixed_lenient": 3594,
|
| 41 |
+
"broken_lenient": 123,
|
| 42 |
+
"net_fixed_lenient": 3471,
|
| 43 |
+
"broken_per_100_correct": 0.04096954597082835,
|
| 44 |
+
"fix_rate_recoverable": 0.059573192063571395,
|
| 45 |
+
"unrecoverable_targets": 966,
|
| 46 |
+
"entity_fix_rate": 0.0,
|
| 47 |
+
"entity_keep_rate": 1.0,
|
| 48 |
+
"entity_spans_whisper_wrong": 57,
|
| 49 |
+
"entity_spans_whisper_right": 217,
|
| 50 |
+
"accepted": 0
|
| 51 |
+
},
|
| 52 |
+
"accept all": {
|
| 53 |
+
"n_examples": 1978,
|
| 54 |
+
"wer": 0.17689930423573352,
|
| 55 |
+
"wer_zwnj_normalized": 0.16549604191112066,
|
| 56 |
+
"exact_match": 0.006572295247724975,
|
| 57 |
+
"hallucination_rate": 0.0,
|
| 58 |
+
"rows_changed": 0.955005055611729,
|
| 59 |
+
"fix_rate": 0.1773377406670531,
|
| 60 |
+
"preservation_rate": 0.9867622927535167,
|
| 61 |
+
"targeted_score": 0.5820500167102849,
|
| 62 |
+
"fix_rate_lenient": 0.1761841522797698,
|
| 63 |
+
"preservation_rate_lenient": 0.9868397824283949,
|
| 64 |
+
"targeted_score_lenient": 0.5815119673540823,
|
| 65 |
+
"fixed_lenient": 10746,
|
| 66 |
+
"broken_lenient": 3951,
|
| 67 |
+
"net_fixed_lenient": 6795,
|
| 68 |
+
"broken_per_100_correct": 1.3160217571605106,
|
| 69 |
+
"fix_rate_recoverable": 0.1784863478101521,
|
| 70 |
+
"unrecoverable_targets": 966,
|
| 71 |
+
"entity_fix_rate": 0.19298245614035087,
|
| 72 |
+
"entity_keep_rate": 0.8894009216589862,
|
| 73 |
+
"entity_spans_whisper_wrong": 57,
|
| 74 |
+
"entity_spans_whisper_right": 217,
|
| 75 |
+
"accepted": 11829
|
| 76 |
+
},
|
| 77 |
+
"oracle": {
|
| 78 |
+
"n_examples": 1978,
|
| 79 |
+
"wer": 0.16575495656293554,
|
| 80 |
+
"wer_zwnj_normalized": 0.15435169423832268,
|
| 81 |
+
"exact_match": 0.007583417593528817,
|
| 82 |
+
"hallucination_rate": 0.0,
|
| 83 |
+
"rows_changed": 0.8923154701718908,
|
| 84 |
+
"fix_rate": 0.17237165729324957,
|
| 85 |
+
"preservation_rate": 0.9992279953612881,
|
| 86 |
+
"targeted_score": 0.5857998263272689,
|
| 87 |
+
"fix_rate_lenient": 0.17277392487662518,
|
| 88 |
+
"preservation_rate_lenient": 0.9993904530965316,
|
| 89 |
+
"targeted_score_lenient": 0.5860821889865784,
|
| 90 |
+
"fixed_lenient": 10538,
|
| 91 |
+
"broken_lenient": 183,
|
| 92 |
+
"net_fixed_lenient": 10355,
|
| 93 |
+
"broken_per_100_correct": 0.06095469034684218,
|
| 94 |
+
"fix_rate_recoverable": 0.1750212404418012,
|
| 95 |
+
"unrecoverable_targets": 966,
|
| 96 |
+
"entity_fix_rate": 0.19298245614035087,
|
| 97 |
+
"entity_keep_rate": 1.0,
|
| 98 |
+
"entity_spans_whisper_wrong": 57,
|
| 99 |
+
"entity_spans_whisper_right": 217,
|
| 100 |
+
"accepted": 5616
|
| 101 |
+
},
|
| 102 |
+
"verifier@0.3": {
|
| 103 |
+
"n_examples": 1978,
|
| 104 |
+
"wer": 0.18431693356911094,
|
| 105 |
+
"wer_zwnj_normalized": 0.17262445806959847,
|
| 106 |
+
"exact_match": 0.006572295247724975,
|
| 107 |
+
"hallucination_rate": 0.0,
|
| 108 |
+
"rows_changed": 0.8427704752275025,
|
| 109 |
+
"fix_rate": 0.0792066799304435,
|
| 110 |
+
"preservation_rate": 0.9964808619715863,
|
| 111 |
+
"targeted_score": 0.5378437709510149,
|
| 112 |
+
"fix_rate_lenient": 0.08350138540488253,
|
| 113 |
+
"preservation_rate_lenient": 0.9964926071620096,
|
| 114 |
+
"targeted_score_lenient": 0.5399969962834461,
|
| 115 |
+
"fixed_lenient": 5093,
|
| 116 |
+
"broken_lenient": 1053,
|
| 117 |
+
"net_fixed_lenient": 4040,
|
| 118 |
+
"broken_per_100_correct": 0.35073928379904273,
|
| 119 |
+
"fix_rate_recoverable": 0.08447865127359355,
|
| 120 |
+
"unrecoverable_targets": 966,
|
| 121 |
+
"entity_fix_rate": 0.15789473684210525,
|
| 122 |
+
"entity_keep_rate": 1.0,
|
| 123 |
+
"entity_spans_whisper_wrong": 57,
|
| 124 |
+
"entity_spans_whisper_right": 217,
|
| 125 |
+
"accepted": 2995
|
| 126 |
+
},
|
| 127 |
+
"verifier@0.7": {
|
| 128 |
+
"n_examples": 1978,
|
| 129 |
+
"wer": 0.18537462746588662,
|
| 130 |
+
"wer_zwnj_normalized": 0.1736215549202047,
|
| 131 |
+
"exact_match": 0.006572295247724975,
|
| 132 |
+
"hallucination_rate": 0.0,
|
| 133 |
+
"rows_changed": 0.7588473205257836,
|
| 134 |
+
"fix_rate": 0.06358779941409616,
|
| 135 |
+
"preservation_rate": 0.9985295149738822,
|
| 136 |
+
"targeted_score": 0.5310586571939891,
|
| 137 |
+
"fix_rate_lenient": 0.06720443329562409,
|
| 138 |
+
"preservation_rate_lenient": 0.998527761030967,
|
| 139 |
+
"targeted_score_lenient": 0.5328660971632955,
|
| 140 |
+
"fixed_lenient": 4099,
|
| 141 |
+
"broken_lenient": 442,
|
| 142 |
+
"net_fixed_lenient": 3657,
|
| 143 |
+
"broken_per_100_correct": 0.14722389690330187,
|
| 144 |
+
"fix_rate_recoverable": 0.06796941376380629,
|
| 145 |
+
"unrecoverable_targets": 966,
|
| 146 |
+
"entity_fix_rate": 0.05263157894736842,
|
| 147 |
+
"entity_keep_rate": 1.0,
|
| 148 |
+
"entity_spans_whisper_wrong": 57,
|
| 149 |
+
"entity_spans_whisper_right": 217,
|
| 150 |
+
"accepted": 1085
|
| 151 |
+
},
|
| 152 |
+
"verifier@0.8": {
|
| 153 |
+
"n_examples": 1978,
|
| 154 |
+
"wer": 0.1857712626771775,
|
| 155 |
+
"wer_zwnj_normalized": 0.17400441807554798,
|
| 156 |
+
"exact_match": 0.006572295247724975,
|
| 157 |
+
"hallucination_rate": 0.0,
|
| 158 |
+
"rows_changed": 0.7254802831142568,
|
| 159 |
+
"fix_rate": 0.05920135353187223,
|
| 160 |
+
"preservation_rate": 0.9991411030642903,
|
| 161 |
+
"targeted_score": 0.5291712282980813,
|
| 162 |
+
"fix_rate_lenient": 0.0626957191808896,
|
| 163 |
+
"preservation_rate_lenient": 0.9991273153622474,
|
| 164 |
+
"targeted_score_lenient": 0.5309115172715685,
|
| 165 |
+
"fixed_lenient": 3824,
|
| 166 |
+
"broken_lenient": 262,
|
| 167 |
+
"net_fixed_lenient": 3562,
|
| 168 |
+
"broken_per_100_correct": 0.08726846377526039,
|
| 169 |
+
"fix_rate_recoverable": 0.06340480117280557,
|
| 170 |
+
"unrecoverable_targets": 966,
|
| 171 |
+
"entity_fix_rate": 0.03508771929824561,
|
| 172 |
+
"entity_keep_rate": 1.0,
|
| 173 |
+
"entity_spans_whisper_wrong": 57,
|
| 174 |
+
"entity_spans_whisper_right": 217,
|
| 175 |
+
"accepted": 479
|
| 176 |
+
},
|
| 177 |
+
"verifier@0.9": {
|
| 178 |
+
"n_examples": 1978,
|
| 179 |
+
"wer": 0.18618993317798455,
|
| 180 |
+
"wer_zwnj_normalized": 0.17440931652040742,
|
| 181 |
+
"exact_match": 0.006572295247724975,
|
| 182 |
+
"hallucination_rate": 0.0,
|
| 183 |
+
"rows_changed": 0.6895854398382204,
|
| 184 |
+
"fix_rate": 0.055895853242053484,
|
| 185 |
+
"preservation_rate": 0.9995555124807417,
|
| 186 |
+
"targeted_score": 0.5277256828613976,
|
| 187 |
+
"fix_rate_lenient": 0.05923630580558425,
|
| 188 |
+
"preservation_rate_lenient": 0.9995370108219557,
|
| 189 |
+
"targeted_score_lenient": 0.52938665831377,
|
| 190 |
+
"fixed_lenient": 3613,
|
| 191 |
+
"broken_lenient": 139,
|
| 192 |
+
"net_fixed_lenient": 3474,
|
| 193 |
+
"broken_per_100_correct": 0.04629891780443204,
|
| 194 |
+
"fix_rate_recoverable": 0.05988971629433422,
|
| 195 |
+
"unrecoverable_targets": 966,
|
| 196 |
+
"entity_fix_rate": 0.0,
|
| 197 |
+
"entity_keep_rate": 1.0,
|
| 198 |
+
"entity_spans_whisper_wrong": 57,
|
| 199 |
+
"entity_spans_whisper_right": 217,
|
| 200 |
+
"accepted": 30
|
| 201 |
+
},
|
| 202 |
+
"verifier@0.95": {
|
| 203 |
+
"n_examples": 1978,
|
| 204 |
+
"wer": 0.18624502140177493,
|
| 205 |
+
"wer_zwnj_normalized": 0.17446440474419783,
|
| 206 |
+
"exact_match": 0.006572295247724975,
|
| 207 |
+
"hallucination_rate": 0.0,
|
| 208 |
+
"rows_changed": 0.6875631951466128,
|
| 209 |
+
"fix_rate": 0.05559820155718829,
|
| 210 |
+
"preservation_rate": 0.9996089846635096,
|
| 211 |
+
"targeted_score": 0.5276035931103489,
|
| 212 |
+
"fix_rate_lenient": 0.05892479464856623,
|
| 213 |
+
"preservation_rate_lenient": 0.9995903045402917,
|
| 214 |
+
"targeted_score_lenient": 0.5292575495944289,
|
| 215 |
+
"fixed_lenient": 3594,
|
| 216 |
+
"broken_lenient": 123,
|
| 217 |
+
"net_fixed_lenient": 3471,
|
| 218 |
+
"broken_per_100_correct": 0.04096954597082835,
|
| 219 |
+
"fix_rate_recoverable": 0.059573192063571395,
|
| 220 |
+
"unrecoverable_targets": 966,
|
| 221 |
+
"entity_fix_rate": 0.0,
|
| 222 |
+
"entity_keep_rate": 1.0,
|
| 223 |
+
"entity_spans_whisper_wrong": 57,
|
| 224 |
+
"entity_spans_whisper_right": 217,
|
| 225 |
+
"accepted": 0
|
| 226 |
+
}
|
| 227 |
+
}
|
verifier-qwen3.5-2b/test_eval/comparison.md
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Candidates: PedramR/ASR_Post-processing/qwen3.5-2b-100pct-masked-weighted-v2/test_eval_best/predictions.jsonl (11829 candidates; skipped rows {'premature_stop': 24})
|
| 2 |
+
|
| 3 |
+
| slice | method | WER | fixed | broken | net | broken/100 | entity fix | accepted |
|
| 4 |
+
|---|---|---|---|---|---|---|---|---|
|
| 5 |
+
| main | whisper | 0.1929 | 0 | 0 | 0 | 0.00 | 0.000 | 0 |
|
| 6 |
+
| main | rules only | 0.1862 | 3594 | 123 | 3471 | 0.04 | 0.000 | 0 |
|
| 7 |
+
| main | accept all | 0.1769 | 10746 | 3951 | 6795 | 1.32 | 0.193 | 11829 |
|
| 8 |
+
| main | oracle | 0.1658 | 10538 | 183 | 10355 | 0.06 | 0.193 | 5616 |
|
| 9 |
+
| main | verifier@0.3 | 0.1843 | 5093 | 1053 | 4040 | 0.35 | 0.158 | 2995 |
|
| 10 |
+
| main | verifier@0.5 | 0.1849 | 4606 | 763 | 3843 | 0.25 | 0.123 | 2091 |
|
| 11 |
+
| main | verifier@0.7 | 0.1854 | 4099 | 442 | 3657 | 0.15 | 0.053 | 1085 |
|
| 12 |
+
| main | verifier@0.8 | 0.1858 | 3824 | 262 | 3562 | 0.09 | 0.035 | 479 |
|
| 13 |
+
| main | verifier@0.9 | 0.1862 | 3613 | 139 | 3474 | 0.05 | 0.000 | 30 |
|
| 14 |
+
| main | verifier@0.95 | 0.1862 | 3594 | 123 | 3471 | 0.04 | 0.000 | 0 |
|
| 15 |
+
| entity | whisper | 0.1866 | 0 | 0 | 0 | 0.00 | 0.000 | 0 |
|
| 16 |
+
| entity | rules only | 0.1800 | 1833 | 49 | 1784 | 0.03 | 0.000 | 0 |
|
| 17 |
+
| entity | accept all | 0.1696 | 5672 | 1970 | 3702 | 1.26 | 0.193 | 6086 |
|
| 18 |
+
| entity | oracle | 0.1587 | 5563 | 98 | 5465 | 0.06 | 0.193 | 2996 |
|
| 19 |
+
| entity | verifier@0.3 | 0.1779 | 2618 | 508 | 2110 | 0.33 | 0.158 | 1510 |
|
| 20 |
+
| entity | verifier@0.5 | 0.1785 | 2359 | 351 | 2008 | 0.23 | 0.123 | 1045 |
|
| 21 |
+
| entity | verifier@0.7 | 0.1792 | 2077 | 207 | 1870 | 0.13 | 0.053 | 543 |
|
| 22 |
+
| entity | verifier@0.8 | 0.1796 | 1956 | 128 | 1828 | 0.08 | 0.035 | 255 |
|
| 23 |
+
| entity | verifier@0.9 | 0.1800 | 1840 | 55 | 1785 | 0.04 | 0.000 | 11 |
|
| 24 |
+
| entity | verifier@0.95 | 0.1800 | 1833 | 49 | 1784 | 0.03 | 0.000 | 0 |
|
| 25 |
+
| typo | whisper | 0.1412 | 0 | 0 | 0 | 0.00 | 0.000 | 0 |
|
| 26 |
+
| typo | rules only | 0.1348 | 345 | 4 | 341 | 0.01 | 0.000 | 0 |
|
| 27 |
+
| typo | accept all | 0.1258 | 1059 | 373 | 686 | 1.09 | 0.222 | 1127 |
|
| 28 |
+
| typo | oracle | 0.1160 | 1035 | 10 | 1025 | 0.03 | 0.222 | 583 |
|
| 29 |
+
| typo | verifier@0.3 | 0.1334 | 462 | 73 | 389 | 0.21 | 0.222 | 242 |
|
| 30 |
+
| typo | verifier@0.5 | 0.1336 | 429 | 42 | 387 | 0.12 | 0.111 | 159 |
|
| 31 |
+
| typo | verifier@0.7 | 0.1342 | 384 | 20 | 364 | 0.06 | 0.000 | 79 |
|
| 32 |
+
| typo | verifier@0.8 | 0.1345 | 361 | 13 | 348 | 0.04 | 0.000 | 35 |
|
| 33 |
+
| typo | verifier@0.9 | 0.1348 | 346 | 4 | 342 | 0.01 | 0.000 | 1 |
|
| 34 |
+
| typo | verifier@0.95 | 0.1348 | 345 | 4 | 341 | 0.01 | 0.000 | 0 |
|
verifier-qwen3.5-2b/test_eval/metrics.json
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"n_examples": 1978,
|
| 3 |
+
"wer": 0.18487057021820447,
|
| 4 |
+
"wer_zwnj_normalized": 0.17314504178441775,
|
| 5 |
+
"exact_match": 0.006572295247724975,
|
| 6 |
+
"hallucination_rate": 0.0,
|
| 7 |
+
"rows_changed": 0.8134479271991911,
|
| 8 |
+
"fix_rate": 0.07160872902730563,
|
| 9 |
+
"preservation_rate": 0.9974366772385628,
|
| 10 |
+
"targeted_score": 0.5345227031329342,
|
| 11 |
+
"fix_rate_lenient": 0.0755168625907891,
|
| 12 |
+
"preservation_rate_lenient": 0.9974585558068503,
|
| 13 |
+
"targeted_score_lenient": 0.5364877091988197,
|
| 14 |
+
"fixed_lenient": 4606,
|
| 15 |
+
"broken_lenient": 763,
|
| 16 |
+
"net_fixed_lenient": 3843,
|
| 17 |
+
"broken_per_100_correct": 0.2541444193149759,
|
| 18 |
+
"fix_rate_recoverable": 0.07639895380412148,
|
| 19 |
+
"unrecoverable_targets": 966,
|
| 20 |
+
"entity_fix_rate": 0.12280701754385964,
|
| 21 |
+
"entity_keep_rate": 1.0,
|
| 22 |
+
"entity_spans_whisper_wrong": 57,
|
| 23 |
+
"entity_spans_whisper_right": 217,
|
| 24 |
+
"accepted": 2091
|
| 25 |
+
}
|
verifier-qwen3.5-2b/test_eval/predictions.jsonl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:76ca4e275fc2d67dfdc79af73025e782a26a1a23e16cfad34071556abb599606
|
| 3 |
+
size 21285375
|
verifier-qwen3.5-2b/test_eval_entity/baselines.json
ADDED
|
@@ -0,0 +1,227 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"whisper": {
|
| 3 |
+
"n_examples": 1010,
|
| 4 |
+
"wer": 0.18660667584675655,
|
| 5 |
+
"wer_zwnj_normalized": 0.17409613475517982,
|
| 6 |
+
"exact_match": 0.0049504950495049506,
|
| 7 |
+
"hallucination_rate": 0.0,
|
| 8 |
+
"rows_changed": 0.0,
|
| 9 |
+
"fix_rate": 0.0,
|
| 10 |
+
"preservation_rate": 1.0,
|
| 11 |
+
"targeted_score": 0.5,
|
| 12 |
+
"fix_rate_lenient": 0.0,
|
| 13 |
+
"preservation_rate_lenient": 1.0,
|
| 14 |
+
"targeted_score_lenient": 0.5,
|
| 15 |
+
"fixed_lenient": 0,
|
| 16 |
+
"broken_lenient": 0,
|
| 17 |
+
"net_fixed_lenient": 0,
|
| 18 |
+
"broken_per_100_correct": 0.0,
|
| 19 |
+
"fix_rate_recoverable": 0.0,
|
| 20 |
+
"unrecoverable_targets": 471,
|
| 21 |
+
"entity_fix_rate": 0.0,
|
| 22 |
+
"entity_keep_rate": 1.0,
|
| 23 |
+
"entity_spans_whisper_wrong": 57,
|
| 24 |
+
"entity_spans_whisper_right": 217,
|
| 25 |
+
"accepted": 0
|
| 26 |
+
},
|
| 27 |
+
"rules only": {
|
| 28 |
+
"n_examples": 1010,
|
| 29 |
+
"wer": 0.18000982056126644,
|
| 30 |
+
"wer_zwnj_normalized": 0.16776614254758168,
|
| 31 |
+
"exact_match": 0.0049504950495049506,
|
| 32 |
+
"hallucination_rate": 0.0,
|
| 33 |
+
"rows_changed": 0.7148514851485148,
|
| 34 |
+
"fix_rate": 0.0566740770892695,
|
| 35 |
+
"preservation_rate": 0.9996911693742963,
|
| 36 |
+
"targeted_score": 0.5281826232317829,
|
| 37 |
+
"fix_rate_lenient": 0.06019704433497537,
|
| 38 |
+
"preservation_rate_lenient": 0.9996856677315474,
|
| 39 |
+
"targeted_score_lenient": 0.5299413560332614,
|
| 40 |
+
"fixed_lenient": 1833,
|
| 41 |
+
"broken_lenient": 49,
|
| 42 |
+
"net_fixed_lenient": 1784,
|
| 43 |
+
"broken_per_100_correct": 0.03143322684525871,
|
| 44 |
+
"fix_rate_recoverable": 0.06077587644684613,
|
| 45 |
+
"unrecoverable_targets": 471,
|
| 46 |
+
"entity_fix_rate": 0.0,
|
| 47 |
+
"entity_keep_rate": 1.0,
|
| 48 |
+
"entity_spans_whisper_wrong": 57,
|
| 49 |
+
"entity_spans_whisper_right": 217,
|
| 50 |
+
"accepted": 0
|
| 51 |
+
},
|
| 52 |
+
"accept all": {
|
| 53 |
+
"n_examples": 1010,
|
| 54 |
+
"wer": 0.16964485861594134,
|
| 55 |
+
"wer_zwnj_normalized": 0.15781214974221028,
|
| 56 |
+
"exact_match": 0.0039603960396039604,
|
| 57 |
+
"hallucination_rate": 0.0,
|
| 58 |
+
"rows_changed": 0.9712871287128713,
|
| 59 |
+
"fix_rate": 0.18689920781538655,
|
| 60 |
+
"preservation_rate": 0.9872092649187711,
|
| 61 |
+
"targeted_score": 0.5870542363670789,
|
| 62 |
+
"fix_rate_lenient": 0.18627257799671593,
|
| 63 |
+
"preservation_rate_lenient": 0.9873625598193552,
|
| 64 |
+
"targeted_score_lenient": 0.5868175689080355,
|
| 65 |
+
"fixed_lenient": 5672,
|
| 66 |
+
"broken_lenient": 1970,
|
| 67 |
+
"net_fixed_lenient": 3702,
|
| 68 |
+
"broken_per_100_correct": 1.263744018064483,
|
| 69 |
+
"fix_rate_recoverable": 0.18859868574668934,
|
| 70 |
+
"unrecoverable_targets": 471,
|
| 71 |
+
"entity_fix_rate": 0.19298245614035087,
|
| 72 |
+
"entity_keep_rate": 0.8894009216589862,
|
| 73 |
+
"entity_spans_whisper_wrong": 57,
|
| 74 |
+
"entity_spans_whisper_right": 217,
|
| 75 |
+
"accepted": 6086
|
| 76 |
+
},
|
| 77 |
+
"oracle": {
|
| 78 |
+
"n_examples": 1010,
|
| 79 |
+
"wer": 0.15873549599171657,
|
| 80 |
+
"wer_zwnj_normalized": 0.14686542628708063,
|
| 81 |
+
"exact_match": 0.0049504950495049506,
|
| 82 |
+
"hallucination_rate": 0.0,
|
| 83 |
+
"rows_changed": 0.9247524752475248,
|
| 84 |
+
"fix_rate": 0.18217114945048063,
|
| 85 |
+
"preservation_rate": 0.9991700176934213,
|
| 86 |
+
"targeted_score": 0.590670583571951,
|
| 87 |
+
"fix_rate_lenient": 0.18269293924466337,
|
| 88 |
+
"preservation_rate_lenient": 0.9993713354630949,
|
| 89 |
+
"targeted_score_lenient": 0.5910321373538792,
|
| 90 |
+
"fixed_lenient": 5563,
|
| 91 |
+
"broken_lenient": 98,
|
| 92 |
+
"net_fixed_lenient": 5465,
|
| 93 |
+
"broken_per_100_correct": 0.06286645369051742,
|
| 94 |
+
"fix_rate_recoverable": 0.1849294506154308,
|
| 95 |
+
"unrecoverable_targets": 471,
|
| 96 |
+
"entity_fix_rate": 0.19298245614035087,
|
| 97 |
+
"entity_keep_rate": 1.0,
|
| 98 |
+
"entity_spans_whisper_wrong": 57,
|
| 99 |
+
"entity_spans_whisper_right": 217,
|
| 100 |
+
"accepted": 2996
|
| 101 |
+
},
|
| 102 |
+
"verifier@0.3": {
|
| 103 |
+
"n_examples": 1010,
|
| 104 |
+
"wer": 0.17786957867657263,
|
| 105 |
+
"wer_zwnj_normalized": 0.16573798315560254,
|
| 106 |
+
"exact_match": 0.0049504950495049506,
|
| 107 |
+
"hallucination_rate": 0.0,
|
| 108 |
+
"rows_changed": 0.8643564356435643,
|
| 109 |
+
"fix_rate": 0.08125371825782009,
|
| 110 |
+
"preservation_rate": 0.9966929387164227,
|
| 111 |
+
"targeted_score": 0.5389733284871214,
|
| 112 |
+
"fix_rate_lenient": 0.08597701149425287,
|
| 113 |
+
"preservation_rate_lenient": 0.9967412083188997,
|
| 114 |
+
"targeted_score_lenient": 0.5413591099065763,
|
| 115 |
+
"fixed_lenient": 2618,
|
| 116 |
+
"broken_lenient": 508,
|
| 117 |
+
"net_fixed_lenient": 2110,
|
| 118 |
+
"broken_per_100_correct": 0.32587916811002915,
|
| 119 |
+
"fix_rate_recoverable": 0.08686080256179325,
|
| 120 |
+
"unrecoverable_targets": 471,
|
| 121 |
+
"entity_fix_rate": 0.15789473684210525,
|
| 122 |
+
"entity_keep_rate": 1.0,
|
| 123 |
+
"entity_spans_whisper_wrong": 57,
|
| 124 |
+
"entity_spans_whisper_right": 217,
|
| 125 |
+
"accepted": 1510
|
| 126 |
+
},
|
| 127 |
+
"verifier@0.7": {
|
| 128 |
+
"n_examples": 1010,
|
| 129 |
+
"wer": 0.17922524311226395,
|
| 130 |
+
"wer_zwnj_normalized": 0.16699757688325273,
|
| 131 |
+
"exact_match": 0.0049504950495049506,
|
| 132 |
+
"hallucination_rate": 0.0,
|
| 133 |
+
"rows_changed": 0.7792079207920792,
|
| 134 |
+
"fix_rate": 0.06437674171024203,
|
| 135 |
+
"preservation_rate": 0.9986424320411774,
|
| 136 |
+
"targeted_score": 0.5315095868757097,
|
| 137 |
+
"fix_rate_lenient": 0.06821018062397373,
|
| 138 |
+
"preservation_rate_lenient": 0.9986721065393942,
|
| 139 |
+
"targeted_score_lenient": 0.5334411435816839,
|
| 140 |
+
"fixed_lenient": 2077,
|
| 141 |
+
"broken_lenient": 207,
|
| 142 |
+
"net_fixed_lenient": 1870,
|
| 143 |
+
"broken_per_100_correct": 0.13278934606058274,
|
| 144 |
+
"fix_rate_recoverable": 0.06888155041862637,
|
| 145 |
+
"unrecoverable_targets": 471,
|
| 146 |
+
"entity_fix_rate": 0.05263157894736842,
|
| 147 |
+
"entity_keep_rate": 1.0,
|
| 148 |
+
"entity_spans_whisper_wrong": 57,
|
| 149 |
+
"entity_spans_whisper_right": 217,
|
| 150 |
+
"accepted": 543
|
| 151 |
+
},
|
| 152 |
+
"verifier@0.8": {
|
| 153 |
+
"n_examples": 1010,
|
| 154 |
+
"wer": 0.1795508160672922,
|
| 155 |
+
"wer_zwnj_normalized": 0.16731781257672312,
|
| 156 |
+
"exact_match": 0.0049504950495049506,
|
| 157 |
+
"hallucination_rate": 0.0,
|
| 158 |
+
"rows_changed": 0.7504950495049505,
|
| 159 |
+
"fix_rate": 0.06052540939975577,
|
| 160 |
+
"preservation_rate": 0.9991828856361589,
|
| 161 |
+
"targeted_score": 0.5298541475179573,
|
| 162 |
+
"fix_rate_lenient": 0.06423645320197044,
|
| 163 |
+
"preservation_rate_lenient": 0.9991788871354708,
|
| 164 |
+
"targeted_score_lenient": 0.5317076701687206,
|
| 165 |
+
"fixed_lenient": 1956,
|
| 166 |
+
"broken_lenient": 128,
|
| 167 |
+
"net_fixed_lenient": 1828,
|
| 168 |
+
"broken_per_100_correct": 0.08211128645292072,
|
| 169 |
+
"fix_rate_recoverable": 0.06487874845725342,
|
| 170 |
+
"unrecoverable_targets": 471,
|
| 171 |
+
"entity_fix_rate": 0.03508771929824561,
|
| 172 |
+
"entity_keep_rate": 1.0,
|
| 173 |
+
"entity_spans_whisper_wrong": 57,
|
| 174 |
+
"entity_spans_whisper_right": 217,
|
| 175 |
+
"accepted": 255
|
| 176 |
+
},
|
| 177 |
+
"verifier@0.9": {
|
| 178 |
+
"n_examples": 1010,
|
| 179 |
+
"wer": 0.17997245973036155,
|
| 180 |
+
"wer_zwnj_normalized": 0.1677287817166768,
|
| 181 |
+
"exact_match": 0.0049504950495049506,
|
| 182 |
+
"hallucination_rate": 0.0,
|
| 183 |
+
"rows_changed": 0.7168316831683168,
|
| 184 |
+
"fix_rate": 0.05689325860287441,
|
| 185 |
+
"preservation_rate": 0.9996525655460833,
|
| 186 |
+
"targeted_score": 0.5282729120744788,
|
| 187 |
+
"fix_rate_lenient": 0.060426929392446635,
|
| 188 |
+
"preservation_rate_lenient": 0.9996471780660227,
|
| 189 |
+
"targeted_score_lenient": 0.5300370537292346,
|
| 190 |
+
"fixed_lenient": 1840,
|
| 191 |
+
"broken_lenient": 55,
|
| 192 |
+
"net_fixed_lenient": 1785,
|
| 193 |
+
"broken_per_100_correct": 0.03528219339773937,
|
| 194 |
+
"fix_rate_recoverable": 0.06100937322792622,
|
| 195 |
+
"unrecoverable_targets": 471,
|
| 196 |
+
"entity_fix_rate": 0.0,
|
| 197 |
+
"entity_keep_rate": 1.0,
|
| 198 |
+
"entity_spans_whisper_wrong": 57,
|
| 199 |
+
"entity_spans_whisper_right": 217,
|
| 200 |
+
"accepted": 11
|
| 201 |
+
},
|
| 202 |
+
"verifier@0.95": {
|
| 203 |
+
"n_examples": 1010,
|
| 204 |
+
"wer": 0.18000982056126644,
|
| 205 |
+
"wer_zwnj_normalized": 0.16776614254758168,
|
| 206 |
+
"exact_match": 0.0049504950495049506,
|
| 207 |
+
"hallucination_rate": 0.0,
|
| 208 |
+
"rows_changed": 0.7148514851485148,
|
| 209 |
+
"fix_rate": 0.0566740770892695,
|
| 210 |
+
"preservation_rate": 0.9996911693742963,
|
| 211 |
+
"targeted_score": 0.5281826232317829,
|
| 212 |
+
"fix_rate_lenient": 0.06019704433497537,
|
| 213 |
+
"preservation_rate_lenient": 0.9996856677315474,
|
| 214 |
+
"targeted_score_lenient": 0.5299413560332614,
|
| 215 |
+
"fixed_lenient": 1833,
|
| 216 |
+
"broken_lenient": 49,
|
| 217 |
+
"net_fixed_lenient": 1784,
|
| 218 |
+
"broken_per_100_correct": 0.03143322684525871,
|
| 219 |
+
"fix_rate_recoverable": 0.06077587644684613,
|
| 220 |
+
"unrecoverable_targets": 471,
|
| 221 |
+
"entity_fix_rate": 0.0,
|
| 222 |
+
"entity_keep_rate": 1.0,
|
| 223 |
+
"entity_spans_whisper_wrong": 57,
|
| 224 |
+
"entity_spans_whisper_right": 217,
|
| 225 |
+
"accepted": 0
|
| 226 |
+
}
|
| 227 |
+
}
|
verifier-qwen3.5-2b/test_eval_entity/metrics.json
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"n_examples": 1010,
|
| 3 |
+
"wer": 0.17846735197105068,
|
| 4 |
+
"wer_zwnj_normalized": 0.16628772109606002,
|
| 5 |
+
"exact_match": 0.0049504950495049506,
|
| 6 |
+
"hallucination_rate": 0.0,
|
| 7 |
+
"rows_changed": 0.8316831683168316,
|
| 8 |
+
"fix_rate": 0.07320662554403982,
|
| 9 |
+
"preservation_rate": 0.9976966382499598,
|
| 10 |
+
"targeted_score": 0.5354516318969998,
|
| 11 |
+
"fix_rate_lenient": 0.0774712643678161,
|
| 12 |
+
"preservation_rate_lenient": 0.9977483545667988,
|
| 13 |
+
"targeted_score_lenient": 0.5376098094673074,
|
| 14 |
+
"fixed_lenient": 2359,
|
| 15 |
+
"broken_lenient": 351,
|
| 16 |
+
"net_fixed_lenient": 2008,
|
| 17 |
+
"broken_per_100_correct": 0.22516454332011854,
|
| 18 |
+
"fix_rate_recoverable": 0.07828813502785283,
|
| 19 |
+
"unrecoverable_targets": 471,
|
| 20 |
+
"entity_fix_rate": 0.12280701754385964,
|
| 21 |
+
"entity_keep_rate": 1.0,
|
| 22 |
+
"entity_spans_whisper_wrong": 57,
|
| 23 |
+
"entity_spans_whisper_right": 217,
|
| 24 |
+
"accepted": 1045
|
| 25 |
+
}
|
verifier-qwen3.5-2b/test_eval_entity/predictions.jsonl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:acb593e8e701d870a3cb3aa4e092c444e15fd012ed72f87bd2975b4b9afcce34
|
| 3 |
+
size 11052274
|
verifier-qwen3.5-2b/test_eval_typo/baselines.json
ADDED
|
@@ -0,0 +1,227 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"whisper": {
|
| 3 |
+
"n_examples": 358,
|
| 4 |
+
"wer": 0.14117586990375533,
|
| 5 |
+
"wer_zwnj_normalized": 0.12749227745015446,
|
| 6 |
+
"exact_match": 0.0,
|
| 7 |
+
"hallucination_rate": 0.0,
|
| 8 |
+
"rows_changed": 0.0,
|
| 9 |
+
"fix_rate": 0.0,
|
| 10 |
+
"preservation_rate": 1.0,
|
| 11 |
+
"targeted_score": 0.5,
|
| 12 |
+
"fix_rate_lenient": 0.0,
|
| 13 |
+
"preservation_rate_lenient": 1.0,
|
| 14 |
+
"targeted_score_lenient": 0.5,
|
| 15 |
+
"fixed_lenient": 0,
|
| 16 |
+
"broken_lenient": 0,
|
| 17 |
+
"net_fixed_lenient": 0,
|
| 18 |
+
"broken_per_100_correct": 0.0,
|
| 19 |
+
"fix_rate_recoverable": 0.0,
|
| 20 |
+
"unrecoverable_targets": 68,
|
| 21 |
+
"entity_fix_rate": 0.0,
|
| 22 |
+
"entity_keep_rate": 1.0,
|
| 23 |
+
"entity_spans_whisper_wrong": 9,
|
| 24 |
+
"entity_spans_whisper_right": 61,
|
| 25 |
+
"accepted": 0
|
| 26 |
+
},
|
| 27 |
+
"rules only": {
|
| 28 |
+
"n_examples": 358,
|
| 29 |
+
"wer": 0.13481912639452656,
|
| 30 |
+
"wer_zwnj_normalized": 0.12139082484491077,
|
| 31 |
+
"exact_match": 0.002793296089385475,
|
| 32 |
+
"hallucination_rate": 0.0,
|
| 33 |
+
"rows_changed": 0.4776536312849162,
|
| 34 |
+
"fix_rate": 0.06630476929042264,
|
| 35 |
+
"preservation_rate": 0.9998823979066828,
|
| 36 |
+
"targeted_score": 0.5330935835985527,
|
| 37 |
+
"fix_rate_lenient": 0.07141378596563859,
|
| 38 |
+
"preservation_rate_lenient": 0.9998827976207917,
|
| 39 |
+
"targeted_score_lenient": 0.5356482917932152,
|
| 40 |
+
"fixed_lenient": 345,
|
| 41 |
+
"broken_lenient": 4,
|
| 42 |
+
"net_fixed_lenient": 341,
|
| 43 |
+
"broken_per_100_correct": 0.011720237920829792,
|
| 44 |
+
"fix_rate_recoverable": 0.07222338862061725,
|
| 45 |
+
"unrecoverable_targets": 68,
|
| 46 |
+
"entity_fix_rate": 0.0,
|
| 47 |
+
"entity_keep_rate": 1.0,
|
| 48 |
+
"entity_spans_whisper_wrong": 9,
|
| 49 |
+
"entity_spans_whisper_right": 61,
|
| 50 |
+
"accepted": 0
|
| 51 |
+
},
|
| 52 |
+
"accept all": {
|
| 53 |
+
"n_examples": 358,
|
| 54 |
+
"wer": 0.12583288657425135,
|
| 55 |
+
"wer_zwnj_normalized": 0.1127619922902147,
|
| 56 |
+
"exact_match": 0.00558659217877095,
|
| 57 |
+
"hallucination_rate": 0.0,
|
| 58 |
+
"rows_changed": 0.888268156424581,
|
| 59 |
+
"fix_rate": 0.216556804963164,
|
| 60 |
+
"preservation_rate": 0.9890336047981654,
|
| 61 |
+
"targeted_score": 0.6027952048806647,
|
| 62 |
+
"fix_rate_lenient": 0.21920927344235147,
|
| 63 |
+
"preservation_rate_lenient": 0.9890708781388262,
|
| 64 |
+
"targeted_score_lenient": 0.6041400757905888,
|
| 65 |
+
"fixed_lenient": 1059,
|
| 66 |
+
"broken_lenient": 373,
|
| 67 |
+
"net_fixed_lenient": 686,
|
| 68 |
+
"broken_per_100_correct": 1.0929121861173783,
|
| 69 |
+
"fix_rate_recoverable": 0.22212891035061935,
|
| 70 |
+
"unrecoverable_targets": 68,
|
| 71 |
+
"entity_fix_rate": 0.2222222222222222,
|
| 72 |
+
"entity_keep_rate": 0.9180327868852459,
|
| 73 |
+
"entity_spans_whisper_wrong": 9,
|
| 74 |
+
"entity_spans_whisper_right": 61,
|
| 75 |
+
"accepted": 1127
|
| 76 |
+
},
|
| 77 |
+
"oracle": {
|
| 78 |
+
"n_examples": 358,
|
| 79 |
+
"wer": 0.11600418677082536,
|
| 80 |
+
"wer_zwnj_normalized": 0.10303540884838273,
|
| 81 |
+
"exact_match": 0.008379888268156424,
|
| 82 |
+
"hallucination_rate": 0.0,
|
| 83 |
+
"rows_changed": 0.7849162011173184,
|
| 84 |
+
"fix_rate": 0.21054672353625437,
|
| 85 |
+
"preservation_rate": 0.9994707905800724,
|
| 86 |
+
"targeted_score": 0.6050087570581634,
|
| 87 |
+
"fix_rate_lenient": 0.21424135789691576,
|
| 88 |
+
"preservation_rate_lenient": 0.9997069940519793,
|
| 89 |
+
"targeted_score_lenient": 0.6069741759744475,
|
| 90 |
+
"fixed_lenient": 1035,
|
| 91 |
+
"broken_lenient": 10,
|
| 92 |
+
"net_fixed_lenient": 1025,
|
| 93 |
+
"broken_per_100_correct": 0.029300594802074482,
|
| 94 |
+
"fix_rate_recoverable": 0.21688011757295822,
|
| 95 |
+
"unrecoverable_targets": 68,
|
| 96 |
+
"entity_fix_rate": 0.2222222222222222,
|
| 97 |
+
"entity_keep_rate": 1.0,
|
| 98 |
+
"entity_spans_whisper_wrong": 9,
|
| 99 |
+
"entity_spans_whisper_right": 61,
|
| 100 |
+
"accepted": 583
|
| 101 |
+
},
|
| 102 |
+
"verifier@0.3": {
|
| 103 |
+
"n_examples": 358,
|
| 104 |
+
"wer": 0.13344055551300707,
|
| 105 |
+
"wer_zwnj_normalized": 0.11998672487299278,
|
| 106 |
+
"exact_match": 0.002793296089385475,
|
| 107 |
+
"hallucination_rate": 0.0,
|
| 108 |
+
"rows_changed": 0.6564245810055865,
|
| 109 |
+
"fix_rate": 0.08918185343156262,
|
| 110 |
+
"preservation_rate": 0.9979125628436186,
|
| 111 |
+
"targeted_score": 0.5435472081375906,
|
| 112 |
+
"fix_rate_lenient": 0.09563237424963776,
|
| 113 |
+
"preservation_rate_lenient": 0.9978610565794486,
|
| 114 |
+
"targeted_score_lenient": 0.5467467154145431,
|
| 115 |
+
"fixed_lenient": 462,
|
| 116 |
+
"broken_lenient": 73,
|
| 117 |
+
"net_fixed_lenient": 389,
|
| 118 |
+
"broken_per_100_correct": 0.2138943420551437,
|
| 119 |
+
"fix_rate_recoverable": 0.09678773882007138,
|
| 120 |
+
"unrecoverable_targets": 68,
|
| 121 |
+
"entity_fix_rate": 0.2222222222222222,
|
| 122 |
+
"entity_keep_rate": 1.0,
|
| 123 |
+
"entity_spans_whisper_wrong": 9,
|
| 124 |
+
"entity_spans_whisper_right": 61,
|
| 125 |
+
"accepted": 242
|
| 126 |
+
},
|
| 127 |
+
"verifier@0.7": {
|
| 128 |
+
"n_examples": 358,
|
| 129 |
+
"wer": 0.13415537004416533,
|
| 130 |
+
"wer_zwnj_normalized": 0.12075259758494805,
|
| 131 |
+
"exact_match": 0.002793296089385475,
|
| 132 |
+
"hallucination_rate": 0.0,
|
| 133 |
+
"rows_changed": 0.5446927374301676,
|
| 134 |
+
"fix_rate": 0.07405971306708026,
|
| 135 |
+
"preservation_rate": 0.9994119895334137,
|
| 136 |
+
"targeted_score": 0.5367358513002469,
|
| 137 |
+
"fix_rate_lenient": 0.07948664872697164,
|
| 138 |
+
"preservation_rate_lenient": 0.9994139881039585,
|
| 139 |
+
"targeted_score_lenient": 0.5394503184154651,
|
| 140 |
+
"fixed_lenient": 384,
|
| 141 |
+
"broken_lenient": 20,
|
| 142 |
+
"net_fixed_lenient": 364,
|
| 143 |
+
"broken_per_100_correct": 0.058601189604148965,
|
| 144 |
+
"fix_rate_recoverable": 0.08041150535376863,
|
| 145 |
+
"unrecoverable_targets": 68,
|
| 146 |
+
"entity_fix_rate": 0.0,
|
| 147 |
+
"entity_keep_rate": 1.0,
|
| 148 |
+
"entity_spans_whisper_wrong": 9,
|
| 149 |
+
"entity_spans_whisper_right": 61,
|
| 150 |
+
"accepted": 79
|
| 151 |
+
},
|
| 152 |
+
"verifier@0.8": {
|
| 153 |
+
"n_examples": 358,
|
| 154 |
+
"wer": 0.13448724821934593,
|
| 155 |
+
"wer_zwnj_normalized": 0.12105894666973016,
|
| 156 |
+
"exact_match": 0.002793296089385475,
|
| 157 |
+
"hallucination_rate": 0.0,
|
| 158 |
+
"rows_changed": 0.505586592178771,
|
| 159 |
+
"fix_rate": 0.06940674680108569,
|
| 160 |
+
"preservation_rate": 0.9996471937200482,
|
| 161 |
+
"targeted_score": 0.534526970260567,
|
| 162 |
+
"fix_rate_lenient": 0.07472572966259573,
|
| 163 |
+
"preservation_rate_lenient": 0.9996190922675731,
|
| 164 |
+
"targeted_score_lenient": 0.5371724109650844,
|
| 165 |
+
"fixed_lenient": 361,
|
| 166 |
+
"broken_lenient": 13,
|
| 167 |
+
"net_fixed_lenient": 348,
|
| 168 |
+
"broken_per_100_correct": 0.03809077324269683,
|
| 169 |
+
"fix_rate_recoverable": 0.07558261599832039,
|
| 170 |
+
"unrecoverable_targets": 68,
|
| 171 |
+
"entity_fix_rate": 0.0,
|
| 172 |
+
"entity_keep_rate": 1.0,
|
| 173 |
+
"entity_spans_whisper_wrong": 9,
|
| 174 |
+
"entity_spans_whisper_right": 61,
|
| 175 |
+
"accepted": 35
|
| 176 |
+
},
|
| 177 |
+
"verifier@0.9": {
|
| 178 |
+
"n_examples": 358,
|
| 179 |
+
"wer": 0.13476806821372955,
|
| 180 |
+
"wer_zwnj_normalized": 0.12133976666411375,
|
| 181 |
+
"exact_match": 0.002793296089385475,
|
| 182 |
+
"hallucination_rate": 0.0,
|
| 183 |
+
"rows_changed": 0.48044692737430167,
|
| 184 |
+
"fix_rate": 0.06649864288483909,
|
| 185 |
+
"preservation_rate": 0.9998823979066828,
|
| 186 |
+
"targeted_score": 0.533190520395761,
|
| 187 |
+
"fix_rate_lenient": 0.07162078244669841,
|
| 188 |
+
"preservation_rate_lenient": 0.9998827976207917,
|
| 189 |
+
"targeted_score_lenient": 0.5357517900337451,
|
| 190 |
+
"fixed_lenient": 346,
|
| 191 |
+
"broken_lenient": 4,
|
| 192 |
+
"net_fixed_lenient": 342,
|
| 193 |
+
"broken_per_100_correct": 0.011720237920829792,
|
| 194 |
+
"fix_rate_recoverable": 0.0724333403317237,
|
| 195 |
+
"unrecoverable_targets": 68,
|
| 196 |
+
"entity_fix_rate": 0.0,
|
| 197 |
+
"entity_keep_rate": 1.0,
|
| 198 |
+
"entity_spans_whisper_wrong": 9,
|
| 199 |
+
"entity_spans_whisper_right": 61,
|
| 200 |
+
"accepted": 1
|
| 201 |
+
},
|
| 202 |
+
"verifier@0.95": {
|
| 203 |
+
"n_examples": 358,
|
| 204 |
+
"wer": 0.13481912639452656,
|
| 205 |
+
"wer_zwnj_normalized": 0.12139082484491077,
|
| 206 |
+
"exact_match": 0.002793296089385475,
|
| 207 |
+
"hallucination_rate": 0.0,
|
| 208 |
+
"rows_changed": 0.4776536312849162,
|
| 209 |
+
"fix_rate": 0.06630476929042264,
|
| 210 |
+
"preservation_rate": 0.9998823979066828,
|
| 211 |
+
"targeted_score": 0.5330935835985527,
|
| 212 |
+
"fix_rate_lenient": 0.07141378596563859,
|
| 213 |
+
"preservation_rate_lenient": 0.9998827976207917,
|
| 214 |
+
"targeted_score_lenient": 0.5356482917932152,
|
| 215 |
+
"fixed_lenient": 345,
|
| 216 |
+
"broken_lenient": 4,
|
| 217 |
+
"net_fixed_lenient": 341,
|
| 218 |
+
"broken_per_100_correct": 0.011720237920829792,
|
| 219 |
+
"fix_rate_recoverable": 0.07222338862061725,
|
| 220 |
+
"unrecoverable_targets": 68,
|
| 221 |
+
"entity_fix_rate": 0.0,
|
| 222 |
+
"entity_keep_rate": 1.0,
|
| 223 |
+
"entity_spans_whisper_wrong": 9,
|
| 224 |
+
"entity_spans_whisper_right": 61,
|
| 225 |
+
"accepted": 0
|
| 226 |
+
}
|
| 227 |
+
}
|
verifier-qwen3.5-2b/test_eval_typo/metrics.json
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"n_examples": 358,
|
| 3 |
+
"wer": 0.13361925914579664,
|
| 4 |
+
"wer_zwnj_normalized": 0.12016542850578234,
|
| 5 |
+
"exact_match": 0.002793296089385475,
|
| 6 |
+
"hallucination_rate": 0.0,
|
| 7 |
+
"rows_changed": 0.611731843575419,
|
| 8 |
+
"fix_rate": 0.08259015122140365,
|
| 9 |
+
"preservation_rate": 0.9987651780201687,
|
| 10 |
+
"targeted_score": 0.5406776646207861,
|
| 11 |
+
"fix_rate_lenient": 0.08880149037466363,
|
| 12 |
+
"preservation_rate_lenient": 0.9987693750183129,
|
| 13 |
+
"targeted_score_lenient": 0.5437854326964883,
|
| 14 |
+
"fixed_lenient": 429,
|
| 15 |
+
"broken_lenient": 42,
|
| 16 |
+
"net_fixed_lenient": 387,
|
| 17 |
+
"broken_per_100_correct": 0.12306249816871283,
|
| 18 |
+
"fix_rate_recoverable": 0.08985933235355868,
|
| 19 |
+
"unrecoverable_targets": 68,
|
| 20 |
+
"entity_fix_rate": 0.1111111111111111,
|
| 21 |
+
"entity_keep_rate": 1.0,
|
| 22 |
+
"entity_spans_whisper_wrong": 9,
|
| 23 |
+
"entity_spans_whisper_right": 61,
|
| 24 |
+
"accepted": 159
|
| 25 |
+
}
|
verifier-qwen3.5-2b/test_eval_typo/predictions.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
verifier-qwen3.5-2b/tokenizer.json
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523
|
| 3 |
+
size 19989325
|
verifier-qwen3.5-2b/tokenizer_config.json
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"add_prefix_space": false,
|
| 3 |
+
"audio_bos_token": "<|audio_start|>",
|
| 4 |
+
"audio_eos_token": "<|audio_end|>",
|
| 5 |
+
"audio_token": "<|audio_pad|>",
|
| 6 |
+
"backend": "tokenizers",
|
| 7 |
+
"bos_token": null,
|
| 8 |
+
"clean_up_tokenization_spaces": false,
|
| 9 |
+
"eos_token": "<|im_end|>",
|
| 10 |
+
"errors": "replace",
|
| 11 |
+
"image_token": "<|image_pad|>",
|
| 12 |
+
"is_local": false,
|
| 13 |
+
"local_files_only": false,
|
| 14 |
+
"model_max_length": 262144,
|
| 15 |
+
"model_specific_special_tokens": {
|
| 16 |
+
"audio_bos_token": "<|audio_start|>",
|
| 17 |
+
"audio_eos_token": "<|audio_end|>",
|
| 18 |
+
"audio_token": "<|audio_pad|>",
|
| 19 |
+
"image_token": "<|image_pad|>",
|
| 20 |
+
"video_token": "<|video_pad|>",
|
| 21 |
+
"vision_bos_token": "<|vision_start|>",
|
| 22 |
+
"vision_eos_token": "<|vision_end|>"
|
| 23 |
+
},
|
| 24 |
+
"pad_token": "<|endoftext|>",
|
| 25 |
+
"pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
|
| 26 |
+
"split_special_tokens": false,
|
| 27 |
+
"tokenizer_class": "Qwen2Tokenizer",
|
| 28 |
+
"unk_token": null,
|
| 29 |
+
"video_token": "<|video_pad|>",
|
| 30 |
+
"vision_bos_token": "<|vision_start|>",
|
| 31 |
+
"vision_eos_token": "<|vision_end|>"
|
| 32 |
+
}
|
verifier-qwen3.5-2b/training_args.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5e0edcdf93ae2070c8051c52e456f0cdeab6475526f2a32adc269e78b335a3bc
|
| 3 |
+
size 5265
|
verifier-qwen3.5-2b/validation_predictions.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
verifier-qwen3.5-2b/verifier_scores.json
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"validation": {
|
| 3 |
+
"examples": 233,
|
| 4 |
+
"yes": 104,
|
| 5 |
+
"auc": 0.8064251639833036,
|
| 6 |
+
"by_threshold": {
|
| 7 |
+
"0.3": {
|
| 8 |
+
"accepted": 101,
|
| 9 |
+
"precision": 0.7029702970297029,
|
| 10 |
+
"recall": 0.6826923076923077,
|
| 11 |
+
"accuracy": 0.7296137339055794
|
| 12 |
+
},
|
| 13 |
+
"0.5": {
|
| 14 |
+
"accepted": 78,
|
| 15 |
+
"precision": 0.7435897435897436,
|
| 16 |
+
"recall": 0.5576923076923077,
|
| 17 |
+
"accuracy": 0.7167381974248928
|
| 18 |
+
},
|
| 19 |
+
"0.7": {
|
| 20 |
+
"accepted": 50,
|
| 21 |
+
"precision": 0.86,
|
| 22 |
+
"recall": 0.41346153846153844,
|
| 23 |
+
"accuracy": 0.7081545064377682
|
| 24 |
+
},
|
| 25 |
+
"0.8": {
|
| 26 |
+
"accepted": 36,
|
| 27 |
+
"precision": 0.9166666666666666,
|
| 28 |
+
"recall": 0.3173076923076923,
|
| 29 |
+
"accuracy": 0.6824034334763949
|
| 30 |
+
},
|
| 31 |
+
"0.9": {
|
| 32 |
+
"accepted": 9,
|
| 33 |
+
"precision": 1.0,
|
| 34 |
+
"recall": 0.08653846153846154,
|
| 35 |
+
"accuracy": 0.592274678111588
|
| 36 |
+
},
|
| 37 |
+
"0.95": {
|
| 38 |
+
"accepted": 0,
|
| 39 |
+
"precision": null,
|
| 40 |
+
"recall": 0.0,
|
| 41 |
+
"accuracy": 0.5536480686695279
|
| 42 |
+
}
|
| 43 |
+
}
|
| 44 |
+
},
|
| 45 |
+
"calibration": {
|
| 46 |
+
"examples": 99,
|
| 47 |
+
"yes": 57,
|
| 48 |
+
"auc": 0.7736006683375104,
|
| 49 |
+
"by_threshold": {
|
| 50 |
+
"0.3": {
|
| 51 |
+
"accepted": 49,
|
| 52 |
+
"precision": 0.7551020408163265,
|
| 53 |
+
"recall": 0.6491228070175439,
|
| 54 |
+
"accuracy": 0.6767676767676768
|
| 55 |
+
},
|
| 56 |
+
"0.5": {
|
| 57 |
+
"accepted": 36,
|
| 58 |
+
"precision": 0.8611111111111112,
|
| 59 |
+
"recall": 0.543859649122807,
|
| 60 |
+
"accuracy": 0.6868686868686869
|
| 61 |
+
},
|
| 62 |
+
"0.7": {
|
| 63 |
+
"accepted": 23,
|
| 64 |
+
"precision": 0.9130434782608695,
|
| 65 |
+
"recall": 0.3684210526315789,
|
| 66 |
+
"accuracy": 0.6161616161616161
|
| 67 |
+
},
|
| 68 |
+
"0.8": {
|
| 69 |
+
"accepted": 8,
|
| 70 |
+
"precision": 0.875,
|
| 71 |
+
"recall": 0.12280701754385964,
|
| 72 |
+
"accuracy": 0.48484848484848486
|
| 73 |
+
},
|
| 74 |
+
"0.9": {
|
| 75 |
+
"accepted": 1,
|
| 76 |
+
"precision": 1.0,
|
| 77 |
+
"recall": 0.017543859649122806,
|
| 78 |
+
"accuracy": 0.43434343434343436
|
| 79 |
+
},
|
| 80 |
+
"0.95": {
|
| 81 |
+
"accepted": 0,
|
| 82 |
+
"precision": null,
|
| 83 |
+
"recall": 0.0,
|
| 84 |
+
"accuracy": 0.42424242424242425
|
| 85 |
+
}
|
| 86 |
+
}
|
| 87 |
+
}
|
| 88 |
+
}
|