Image-Text-to-Text
Transformers
Safetensors
mage_vl
multimodal
vision-language-model
mage-vl
video-understanding
streaming
conversational
custom_code
Instructions to use microsoft/Mage-VL with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use microsoft/Mage-VL with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("image-text-to-text", model="microsoft/Mage-VL", trust_remote_code=True) messages = [ { "role": "user", "content": [ {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"}, {"type": "text", "text": "What animal is on the candy?"} ] }, ] pipe(text=messages)# Load model directly from transformers import AutoModelForImageTextToText model = AutoModelForImageTextToText.from_pretrained("microsoft/Mage-VL", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use microsoft/Mage-VL with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "microsoft/Mage-VL" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "microsoft/Mage-VL", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }'Use Docker
docker model run hf.co/microsoft/Mage-VL
- SGLang
How to use microsoft/Mage-VL with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "microsoft/Mage-VL" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "microsoft/Mage-VL", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "microsoft/Mage-VL" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "microsoft/Mage-VL", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }' - Docker Model Runner
How to use microsoft/Mage-VL with Docker Model Runner:
docker model run hf.co/microsoft/Mage-VL
| # Copyright (c) Microsoft Corporation. | |
| # Licensed under the MIT License. | |
| import os | |
| import numpy as np | |
| from PIL import Image | |
| class PNGReader(): | |
| def __init__(self, src_path, width, height, start_num=1): | |
| self.eof = False | |
| self.src_path = src_path | |
| self.width = width | |
| self.height = height | |
| pngs = os.listdir(self.src_path) | |
| if 'im1.png' in pngs: | |
| self.padding = 1 | |
| elif 'im00001.png' in pngs: | |
| self.padding = 5 | |
| else: | |
| raise ValueError('unknown image naming convention; please specify') | |
| self.current_frame_index = start_num | |
| def read_one_frame(self): | |
| # rgb: 3xhxw uint8 numpy array | |
| if self.eof: | |
| return None | |
| png_path = os.path.join(self.src_path, | |
| f"im{str(self.current_frame_index).zfill(self.padding)}.png" | |
| ) | |
| if not os.path.exists(png_path): | |
| self.eof = True | |
| return None | |
| rgb = Image.open(png_path).convert('RGB') | |
| rgb = np.asarray(rgb).astype(np.uint8).transpose(2, 0, 1) | |
| _, height, width = rgb.shape | |
| assert height == self.height | |
| assert width == self.width | |
| self.current_frame_index += 1 | |
| return rgb | |
| def close(self): | |
| self.current_frame_index = 1 | |
| class YUV420Reader(): | |
| def __init__(self, src_path, width, height, skip_frame=0): | |
| self.eof = False | |
| if not src_path.endswith('.yuv'): | |
| src_path = src_path + '.yuv' | |
| self.src_path = src_path | |
| self.y_size = width * height | |
| self.y_width = width | |
| self.y_height = height | |
| self.uv_size = width * height // 2 | |
| self.uv_width = width // 2 | |
| self.uv_height = height // 2 | |
| # pylint: disable=R1732 | |
| self.file = open(src_path, "rb") | |
| # pylint: enable=R1732 | |
| skipped_frame = 0 | |
| while not self.eof and skipped_frame < skip_frame: | |
| y = self.file.read(self.y_size) | |
| uv = self.file.read(self.uv_size) | |
| if not y or not uv: | |
| self.eof = True | |
| skipped_frame += 1 | |
| def read_one_frame(self): | |
| # y: 1xhxw uint8 numpy array | |
| # uv: 2x(h/2)x(w/2) uint8 numpy array | |
| if self.eof: | |
| return None, None | |
| y = self.file.read(self.y_size) | |
| uv = self.file.read(self.uv_size) | |
| if not y or not uv: | |
| self.eof = True | |
| return None, None | |
| y = np.frombuffer(y, dtype=np.uint8).copy().reshape(1, self.y_height, self.y_width) | |
| uv = np.frombuffer(uv, dtype=np.uint8).copy().reshape(2, self.uv_height, self.uv_width) | |
| return y, uv | |
| def close(self): | |
| self.file.close() | |