Video-Text-to-Text
Transformers
Safetensors
English
Chinese
moss_vl
feature-extraction
Realtime
Streaming
Video-Understanding
Image-Understanding
MOSS-VL
OpenMOSS
multimodal
video
vision-language
custom_code
Instructions to use OpenMOSS-Team/MOSS-VL-Realtime with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use OpenMOSS-Team/MOSS-VL-Realtime with Transformers:
# Load model directly from transformers import AutoModel model = AutoModel.from_pretrained("OpenMOSS-Team/MOSS-VL-Realtime", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
simplify: processing reduced to stock + only the <=8-D device-aware 10-D routing
Browse filesPer review: the signature defaults are never exercised
(the framework setdefaults every valid kwarg and passes them explicitly) and the
explicit interpolation chain was redundant (parent class already defaults to
BICUBIC via resample; pixel_values verified bitwise-identical). processing now
matches the video_processing minimal style. preprocessor_config.json is reverted
to stock in the next commit.
- processing_moss_vl.py +15 -19
processing_moss_vl.py
CHANGED
|
@@ -49,28 +49,24 @@ class MossVLImageProcessorFast(Qwen2VLImageProcessorFast):
|
|
| 49 |
"""
|
| 50 |
# Multi-image batch total pixels limit (read from config)
|
| 51 |
multi_image_max_pixels = None
|
| 52 |
-
|
| 53 |
-
def __init__(self, *args, **kwargs):
|
| 54 |
-
super().__init__(*args, **kwargs)
|
| 55 |
-
if not hasattr(self, 'interpolation') or self.interpolation is None:
|
| 56 |
-
self.interpolation = "BICUBIC"
|
| 57 |
|
| 58 |
def _preprocess(
|
| 59 |
self,
|
| 60 |
images: list["torch.Tensor"],
|
| 61 |
do_resize: bool,
|
| 62 |
size: SizeDict,
|
| 63 |
-
interpolation: Optional["F.InterpolationMode"]
|
| 64 |
-
do_rescale: bool
|
| 65 |
-
rescale_factor: float
|
| 66 |
-
do_normalize: bool
|
| 67 |
-
image_mean: Optional[Union[float, list[float]]]
|
| 68 |
-
image_std: Optional[Union[float, list[float]]]
|
| 69 |
-
patch_size: int
|
| 70 |
-
temporal_patch_size: int
|
| 71 |
-
merge_size: int
|
| 72 |
-
disable_grouping: Optional[bool]
|
| 73 |
-
return_tensors: Optional[Union[str, TensorType]]
|
| 74 |
**kwargs,
|
| 75 |
):
|
| 76 |
"""Override _preprocess to use custom smart_resize with batch-level max_pixels.
|
|
@@ -168,8 +164,8 @@ class MossVLImageProcessorFast(Qwen2VLImageProcessorFast):
|
|
| 168 |
)
|
| 169 |
# Reorder dimensions to group grid and patch information for subsequent flattening.
|
| 170 |
# (batch, grid_t, grid_h, grid_w, merge_h, merge_w, channel, temp_patch_size, patch_h, patch_w)
|
| 171 |
-
# NPU
|
| 172 |
-
# CPU there. CUDA handles
|
| 173 |
patches_device = patches.device
|
| 174 |
if patches_device.type == "npu":
|
| 175 |
patches = patches.cpu()
|
|
@@ -281,7 +277,7 @@ class MossVLImagesKwargs(ImagesKwargs):
|
|
| 281 |
patch_size: Optional[int]
|
| 282 |
temporal_patch_size: Optional[int]
|
| 283 |
merge_size: Optional[int]
|
| 284 |
-
|
| 285 |
|
| 286 |
|
| 287 |
class MossVLVideosKwargs(VideosKwargs, total=False):
|
|
|
|
| 49 |
"""
|
| 50 |
# Multi-image batch total pixels limit (read from config)
|
| 51 |
multi_image_max_pixels = None
|
| 52 |
+
|
|
|
|
|
|
|
|
|
|
|
|
|
| 53 |
|
| 54 |
def _preprocess(
|
| 55 |
self,
|
| 56 |
images: list["torch.Tensor"],
|
| 57 |
do_resize: bool,
|
| 58 |
size: SizeDict,
|
| 59 |
+
interpolation: Optional["F.InterpolationMode"],
|
| 60 |
+
do_rescale: bool,
|
| 61 |
+
rescale_factor: float,
|
| 62 |
+
do_normalize: bool,
|
| 63 |
+
image_mean: Optional[Union[float, list[float]]],
|
| 64 |
+
image_std: Optional[Union[float, list[float]]],
|
| 65 |
+
patch_size: int,
|
| 66 |
+
temporal_patch_size: int,
|
| 67 |
+
merge_size: int,
|
| 68 |
+
disable_grouping: Optional[bool],
|
| 69 |
+
return_tensors: Optional[Union[str, TensorType]],
|
| 70 |
**kwargs,
|
| 71 |
):
|
| 72 |
"""Override _preprocess to use custom smart_resize with batch-level max_pixels.
|
|
|
|
| 164 |
)
|
| 165 |
# Reorder dimensions to group grid and patch information for subsequent flattening.
|
| 166 |
# (batch, grid_t, grid_h, grid_w, merge_h, merge_w, channel, temp_patch_size, patch_h, patch_w)
|
| 167 |
+
# NPU ops support at most 8-D tensors; route the 10-D permute+reshape
|
| 168 |
+
# through CPU there. CUDA handles 10-D natively — keep it on-device.
|
| 169 |
patches_device = patches.device
|
| 170 |
if patches_device.type == "npu":
|
| 171 |
patches = patches.cpu()
|
|
|
|
| 277 |
patch_size: Optional[int]
|
| 278 |
temporal_patch_size: Optional[int]
|
| 279 |
merge_size: Optional[int]
|
| 280 |
+
|
| 281 |
|
| 282 |
|
| 283 |
class MossVLVideosKwargs(VideosKwargs, total=False):
|