Release PathoSynVLM model weights

.gitattributes

@@ -1,35 +1,6 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
 *.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
 *.pt filter=lfs diff=lfs merge=lfs -text
 *.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+tokenizer/tokenizer.json filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,21 @@
+Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International
+
+Copyright (c) 2026 PathoSynVLM authors
+
+This repository is licensed under the Creative Commons Attribution-NonCommercial-ShareAlike
+4.0 International Public License (CC BY-NC-SA 4.0).
+
+SPDX-License-Identifier: CC-BY-NC-SA-4.0
+
+You are free to share and adapt the licensed material for non-commercial purposes,
+provided that you give appropriate credit, indicate changes, and distribute adapted
+material under the same license.
+
+Full legal code:
+https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode
+
+Human-readable summary:
+https://creativecommons.org/licenses/by-nc-sa/4.0/
+
+Dataset access, pretrained third-party models, and any externally hosted model
+weights remain subject to their own licenses and terms of use.

README.md

@@ -1,3 +1,232 @@
 ---
 license: cc-by-nc-sa-4.0
+language:
+- en
+library_name: transformers
+pipeline_tag: image-text-to-text
+base_model:
+- Qwen/Qwen2.5-3B-Instruct
+tags:
+- pathology
+- computational-pathology
+- digital-pathology
+- histopathology
+- whole-slide-image
+- vision-language-model
+- report-generation
+- synoptic-report
+- case-level
+- conch
+- qwen2.5
+datasets:
+- david4real/HistGen
+metrics:
+- rouge
+- meteor
+- bleu
+- bertscore
+arxiv: 2605.30716
+model-index:
+- name: PathoSynVLM
+  results:
+  - task:
+      type: image-text-to-text
+      name: Case-level pathology synoptic report generation
+    dataset:
+      name: HISTAI case-report pairs
+      type: HISTAI
+    metrics:
+    - type: rouge
+      name: ROUGE-L
+      value: 0.2495
+    - type: meteor
+      name: METEOR
+      value: 0.1988
+    - type: bleu
+      name: BLEU-4
+      value: 0.0525
+    - type: bertscore
+      name: BERTScore F1
+      value: 0.3018
 ---
+
+# PathoSynVLM: Case-Level Pathology Synoptic Report Generation
+
+PathoSynVLM is a token-efficient vision-language model for generating
+case-level pathology synoptic reports from one or more whole-slide
+images represented as precomputed CONCHv1.5 patch embeddings.
+
+Paper: [Simple Token-Efficient Vision-Language Model for Case-level
+Pathology Synoptic Report Generation](https://arxiv.org/abs/2605.30716)
+
+Code: [https://github.com/AtlasAnalyticsLab/PathoSynVLM](https://github.com/AtlasAnalyticsLab/PathoSynVLM)
+
+Model: [https://huggingface.co/AtlasAnalyticsLab/PathoSynVLM](https://huggingface.co/AtlasAnalyticsLab/PathoSynVLM)
+
+![PathoSynVLM architecture](assets/paper_architecture.png)
+
+## What This Repository Contains
+
+This Hugging Face repository contains the exported Stage 2 PathoSynVLM
+model package:
+
+```text
+README.md
+LICENSE
+model_index.json
+config.json
+vlm_state.pt
+labels.json
+tokenizer/
+llm/
+best_checkpoint_summary.json
+assets/
+examples/
+```
+
+The paper run used `unfreeze_llm_base=true`, so the release package
+includes the merged/full language-model weights under `llm/`, not only a
+LoRA adapter.
+
+## Quickstart
+
+Install the code repository:
+
+```bash
+git clone https://github.com/AtlasAnalyticsLab/PathoSynVLM PathoSynVLM
+cd PathoSynVLM
+conda create -n pathosynvlm python=3.11 -y
+conda activate pathosynvlm
+export PYTHONNOUSERSITE=1
+pip install -e .
+```
+
+Download this model repository:
+
+```bash
+source configs/paths.example.env
+hf download AtlasAnalyticsLab/PathoSynVLM --local-dir "$PATHOSYNVLM_WEIGHTS_ROOT/pathosynvlm-stage2-main"
+```
+
+Generate a case-level report from one or more slide embedding files:
+
+```bash
+python scripts/generate_case_report.py \
+  --embeddings slide_1.h5 slide_2.h5 \
+  --output_json report.json
+```
+
+Relative `--embeddings` paths are resolved under
+`PATHOSYNVLM_EMBEDDINGS_ROOT`. Absolute `.h5` paths also work.
+
+The output follows:
+
+```text
+Diagnosis: ...
+Certainty: ...
+Conclusion: ...
+```
+
+## Input Format
+
+PathoSynVLM runs on precomputed WSI patch embeddings, not raw WSIs. Each
+`.h5` file should contain:
+
+```text
+/features/conch_v15  # shape: (num_patches, 768)
+```
+
+Use the code repository docs for dataset placement, metadata filtering,
+and embedding generation details.
+
+### From Precomputed H5 Feature Files
+
+This is the fastest path. Put one or more WSI embedding files for a case
+into the `--embeddings` argument:
+
+```bash
+python scripts/generate_case_report.py \
+  --embeddings case_slide_1.h5 case_slide_2.h5 \
+  --output_json report.json
+```
+
+### From Raw Whole-Slide Images
+
+First extract tissue patches and CONCHv1.5 patch embeddings using a WSI
+preprocessing pipeline that writes the H5 layout above. Then pass the
+resulting H5 files to `scripts/generate_case_report.py`.
+
+## Running The Paper Pipeline
+
+To rerun training and evaluation, follow the code repository guide:
+
+```bash
+git clone https://github.com/AtlasAnalyticsLab/PathoSynVLM PathoSynVLM
+cd PathoSynVLM
+export PYTHONNOUSERSITE=1
+pip install -e .
+
+# 1. Download HistGen, REG2025, and HISTAI metadata/WSIs.
+# 2. Extract CONCHv1.5 H5 patch embeddings.
+# 3. Prepare Stage 1 and Stage 2 metadata.
+# 4. Train Stage 1 alignment.
+# 5. Train Stage 2 HISTAI report generation.
+# 6. Evaluate with scripts/evaluate_checkpoint.py.
+```
+
+## Training Recipe
+
+- Stage 1: train the two-layer MLP aligner on HistGen + REG2025 while
+  keeping the CONCHv1.5 patch encoder and LLM frozen.
+- Stage 2: finetune on HISTAI case-report pairs with WSI marker tokens.
+
+Checkpoint selected for release:
+
+- checkpoint step: `30400`
+- checkpoint epoch: `7`
+- validation loss: `1.010892`
+- prompt style: `double`
+- patch level: `5x_512`
+- max vision tokens: `4096`
+
+## Reported Metrics
+
+Stage 1 aligner-only training:
+
+| ROUGE-L | METEOR | BLEU-4 | BERTScore F1 |
+|---:|---:|---:|---:|
+| 0.4743 | 0.4810 | 0.1247 | 0.4253 |
+
+Stage 2 HISTAI main result:
+
+| ROUGE-L | METEOR | BLEU-4 | BERTScore F1 | Diagnosis Exact | Diagnosis Relaxed | Certainty |
+|---:|---:|---:|---:|---:|---:|---:|
+| 0.2495 | 0.1988 | 0.0525 | 0.3018 | 0.1667 | 0.3333 | 0.9000 |
+
+## Intended Use
+
+This model is intended for research on pathology report generation from
+precomputed WSI patch embeddings.
+
+It is not a clinical diagnostic device and should not be used for patient
+care without appropriate validation, regulatory review, and expert
+oversight.
+
+## License and Commercial Use
+
+This repository uses CC BY-NC-SA 4.0. Research and non-commercial use only.
+Dataset access, pretrained third-party models, and any externally hosted
+model weights remain subject to their own terms.
+
+## Citation
+
+```bibtex
+@inproceedings{yang2026simpletokenvlm,
+  title     = {Simple Token-Efficient Vision-Language Model for Case-Level Pathology Synoptic Report Generation},
+  author    = {Yang, Zhiyuan and Cheng, Jiahao and Trinh, Vincent Quoc-Huy and Hosseini, Mahdi S.},
+  booktitle = {Proceedings of the 7th International Conference on Deep Learning Theory and Applications},
+  pages     = {514--537},
+  year      = {2026},
+  issn      = {2184-9277}
+}
+```

best_checkpoint_summary.json

@@ -0,0 +1,39 @@
+{
+  "best_step": 30400,
+  "best_epoch": 7,
+  "best_source": "step_eval",
+  "best_val_loss": 1.0108920872211455,
+  "metrics": {
+    "val_loss": 1.0108920872211455,
+    "val_n_samples": 30,
+    "val/vision_tokens_total": 34833,
+    "val/text_tokens_total": 9942,
+    "val/target_tokens_total": 3305,
+    "val/wsi_count_total": 188,
+    "val/avg_vision_tokens_per_sample": 1161.1,
+    "val/avg_text_tokens_per_sample": 331.4,
+    "val/avg_target_tokens_per_sample": 110.16666666666667,
+    "val/avg_wsi_per_sample": 6.266666666666667,
+    "val_bertscore_f1": 0.3017574870338043,
+    "val/field_eval_n": 30,
+    "val_diagnosis_relaxed_match_rate": 0.3333333333333333,
+    "val_diagnosis_exact_match_rate": 0.16666666666666666,
+    "val_certainty_match_rate": 0.9,
+    "val_certainty_exact_match_rate": 0.9,
+    "val_conclusion_exact_match_rate": 0.0,
+    "val_conclusion_rougeL": 0.24946812913260996,
+    "val_conclusion_meteor": 0.1988453811627603,
+    "val_conclusion_bleu4": 5.251211118089842,
+    "val/pred_format_score_0_count": 0,
+    "val/ref_format_score_0_count": 0,
+    "val/pred_format_score_1_count": 0,
+    "val/ref_format_score_1_count": 0,
+    "val/pred_format_score_2_count": 0,
+    "val/ref_format_score_2_count": 0,
+    "val/pred_format_score_3_count": 30,
+    "val/ref_format_score_3_count": 30,
+    "val/pred_diagnosis_present_rate": 1.0,
+    "val/pred_certainty_present_rate": 1.0,
+    "val/pred_conclusion_present_rate": 1.0
+  }
+}

config.json

@@ -0,0 +1,31 @@
+{
+  "format_version": 1,
+  "checkpoint_step": 30400,
+  "base_llm": "Qwen/Qwen2.5-3B-Instruct",
+  "llm_kind": "merged",
+  "llm_path": "llm",
+  "vision_dim": 768,
+  "feature_key": "conch_v15",
+  "patch_level": "5x_512",
+  "use_wsi_markers": true,
+  "use_wsi_index_emb": true,
+  "prompt_style": "double",
+  "report_target_field": "conclusion",
+  "report_target_label": "",
+  "training_config": {
+    "llm": "Qwen/Qwen2.5-3B-Instruct",
+    "vision_dim": 768,
+    "feature_key": "conch_v15",
+    "patch_level": "5x_512",
+    "use_wsi_markers": true,
+    "prompt_style": "double",
+    "max_text_length": 384,
+    "max_vision_tokens": 4096,
+    "vision_token_dropout": 0.0,
+    "lora_r": 16,
+    "lora_alpha": 32,
+    "lora_dropout": 0.05,
+    "lora_target": "q_proj,k_proj,v_proj,o_proj",
+    "unfreeze_llm_base": true
+  }
+}

examples/case_input_example.json

@@ -0,0 +1,9 @@
+{
+  "case_id": "example_case",
+  "slide_embeddings": [
+    "HISTAI-example/conch_v15/5x_512/patches/slide_1.h5",
+    "HISTAI-example/conch_v15/5x_512/patches/slide_2.h5"
+  ],
+  "feature_key": "conch_v15",
+  "command": "python scripts/generate_case_report.py --embeddings HISTAI-example/conch_v15/5x_512/patches/slide_1.h5 HISTAI-example/conch_v15/5x_512/patches/slide_2.h5 --output_json report.json"
+}

labels.json

@@ -0,0 +1,30 @@
+{
+  "task": "case_level_pathology_synoptic_report_generation",
+  "input": {
+    "type": "one_or_more_h5_wsi_embedding_files",
+    "feature_dataset": "/features/conch_v15",
+    "feature_dim": 768,
+    "patch_level": "5x_512"
+  },
+  "output_fields": [
+    {
+      "name": "diagnosis",
+      "label": "Diagnosis",
+      "type": "free_text",
+      "required": true
+    },
+    {
+      "name": "certainty",
+      "label": "Certainty",
+      "type": "free_text_or_percentage",
+      "required": true
+    },
+    {
+      "name": "conclusion",
+      "label": "Conclusion",
+      "type": "free_text",
+      "required": true
+    }
+  ],
+  "expected_text_format": "Diagnosis: ...\\nCertainty: ...\\nConclusion: ..."
+}

llm/config.json

@@ -0,0 +1,69 @@
+{
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "dtype": "bfloat16",
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 32768,
+  "max_window_layers": 70,
+  "model_type": "qwen2",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 36,
+  "num_key_value_heads": 2,
+  "pad_token_id": null,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 1000000.0,
+    "rope_type": "default"
+  },
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.0.0",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}

llm/generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.05,
+  "temperature": 0.7,
+  "top_k": 20,
+  "top_p": 0.8,
+  "transformers_version": "5.0.0"
+}

llm/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa8a241416b1b9fcc6c2a3557ea914148c76abbdee6b403e2232aaad90ac859c
+size 6171927112

model_index.json

@@ -0,0 +1,52 @@
+{
+  "name": "PathoSynVLM",
+  "results": [
+    {
+      "task": {
+        "type": "image-text-to-text",
+        "name": "Case-level pathology synoptic report generation"
+      },
+      "dataset": {
+        "name": "HISTAI case-report pairs",
+        "type": "HISTAI"
+      },
+      "metrics": [
+        {
+          "type": "rouge",
+          "name": "ROUGE-L",
+          "value": 0.2495
+        },
+        {
+          "type": "meteor",
+          "name": "METEOR",
+          "value": 0.1988
+        },
+        {
+          "type": "bleu",
+          "name": "BLEU-4",
+          "value": 0.0525
+        },
+        {
+          "type": "bertscore",
+          "name": "BERTScore F1",
+          "value": 0.3018
+        },
+        {
+          "type": "accuracy",
+          "name": "Diagnosis Exact",
+          "value": 0.1667
+        },
+        {
+          "type": "accuracy",
+          "name": "Diagnosis Relaxed",
+          "value": 0.3333
+        },
+        {
+          "type": "accuracy",
+          "name": "Certainty",
+          "value": 0.9
+        }
+      ]
+    }
+  ]
+}

tokenizer/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

tokenizer/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
+size 11421892

tokenizer/tokenizer_config.json

@@ -0,0 +1,29 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": true,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": null
+}

vlm_state.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b66e6db4fefb9efb6308db3d5d189395b8ecd8f56f3179c36c0477f158b9dc3
+size 46443749