scripts/vis_psalm_multiscale.py

import argparse
import json
#import tqdm
import cv2
import os
import numpy as np
from pycocotools import mask as mask_utils
import random
from PIL import Image
from natsort import natsorted

EVALMODE = "test"


def blend_mask(input_img, binary_mask, alpha=0.5):
    if input_img.ndim == 2:
        return input_img
    mask_image = np.zeros(input_img.shape, np.uint8)
    mask_image[:, :, 1] = 255
    mask_image = mask_image * np.repeat(binary_mask[:, :, np.newaxis], 3, axis=2)
    blend_image = input_img[:, :, :].copy()
    pos_idx = binary_mask > 0
    for ind in range(input_img.ndim):
        ch_img1 = input_img[:, :, ind]
        ch_img2 = mask_image[:, :, ind]
        ch_img3 = blend_image[:, :, ind]
        ch_img3[pos_idx] = alpha * ch_img1[pos_idx] + (1 - alpha) * ch_img2[pos_idx]
        blend_image[:, :, ind] = ch_img3
    return blend_image


def upsample_mask(mask, frame):
    H, W = frame.shape[:2]
    mH, mW = mask.shape[:2]

    if W > H:
        ratio = mW / W
        h = H * ratio
        diff = int((mH - h) // 2)
        if diff == 0:
            mask = mask
        else:
            mask = mask[diff:-diff]
    else:
        ratio = mH / H
        w = W * ratio
        diff = int((mW - w) // 2)
        if diff == 0:
            mask = mask
        else:
            mask = mask[:, diff:-diff]

    mask = cv2.resize(mask, (W, H))
    return mask


def downsample(mask, frame):
    H, W = frame.shape[:2]
    mH, mW = mask.shape[:2]

    mask = cv2.resize(mask, (W, H))
    return mask


from PIL import Image, ImageDraw
import numpy as np
import cv2


def scale_mask_object(img, mask, scale_factor):
    """
    Scales the object in the mask by a given factor and applies the scaled mask onto the image.
    
    Parameters:
    img (PIL.Image or numpy array): The original image.
    mask (PIL.Image or numpy array): The COCO mask, where non-zero regions represent the object.
    scale_factor (float): The scaling factor (e.g., 2.0 for doubling, 0.5 for half).
    
    Returns:
    new_img (PIL.Image): The modified image with the scaled object.
    new_mask (PIL.Image): The modified mask with the scaled object.
    """
    # Convert PIL images to numpy arrays if necessary
    if isinstance(img, Image.Image):
        img = np.array(img)
    if isinstance(mask, Image.Image):
        mask = np.array(mask)

    # Get bounding box of the object in the mask
    y, x = np.where(mask > 0)  # Find all non-zero points
    if len(x) == 0 or len(y) == 0:
        raise ValueError("No object found in the mask.")

    xmin, xmax = x.min(), x.max()
    ymin, ymax = y.min(), y.max()
    object_crop = mask[ymin:ymax+1, xmin:xmax+1]  # Crop the object from the mask

    # Scale the cropped object mask
    obj_height, obj_width = object_crop.shape[:2]
    new_obj_height = int(obj_height * scale_factor)
    new_obj_width = int(obj_width * scale_factor)
    scaled_object_crop = cv2.resize(object_crop, (new_obj_width, new_obj_height), interpolation=cv2.INTER_NEAREST)

    # Scale the cropped object from the original image
    img_object_crop = img[ymin:ymax+1, xmin:xmax+1]
    scaled_img_object_crop = cv2.resize(img_object_crop, (new_obj_width, new_obj_height), interpolation=cv2.INTER_LINEAR)

    # Calculate new positions
    center_x = (xmin + xmax) // 2
    center_y = (ymin + ymax) // 2
    new_xmin = max(center_x - new_obj_width // 2, 0)
    new_ymin = max(center_y - new_obj_height // 2, 0)
    new_xmax = min(new_xmin + new_obj_width, img.shape[1])
    new_ymax = min(new_ymin + new_obj_height, img.shape[0])

    # Create new mask and image with the scaled object
    new_mask = np.zeros_like(mask)
    new_mask[new_ymin:new_ymax, new_xmin:new_xmax] = scaled_object_crop[:new_ymax-new_ymin, :new_xmax-new_xmin]

    new_img = img.copy()
    new_img[new_ymin:new_ymax, new_xmin:new_xmax] = scaled_img_object_crop[:new_ymax-new_ymin, :new_xmax-new_xmin]

    # Convert back to PIL images if needed
    #new_img = Image.fromarray(new_img)
    #new_mask = Image.fromarray(new_mask)

    return new_img, new_mask


from PIL import Image
import numpy as np
import cv2

def scale_mask_object_with_background(img, mask, scale_factor, padding=0.25):
    """
    Scales the object in the mask by a given factor and adjusts the background region accordingly.
    
    Parameters:
    img (PIL.Image or numpy array): The original image.
    mask (PIL.Image or numpy array): The binary mask image where non-zero regions represent the object.
    scale_factor (float): Scaling factor (e.g., 2.0 for double, 0.5 for half).
    padding (float): Fractional padding to include around the object during scaling. For example, 0.25 adds 25% padding.
    
    Returns:
    new_img (PIL.Image): The modified image with the scaled object and adjusted background.
    new_mask (PIL.Image): The modified mask with the scaled object and adjusted background.
    """
    # Convert PIL images to numpy arrays if necessary
    if isinstance(img, Image.Image):
        img = np.array(img)
    if isinstance(mask, Image.Image):
        mask = np.array(mask)
    
    # Get bounding box of the object in the mask
    y, x = np.where(mask > 0)  # Find all non-zero points
    if len(x) == 0 or len(y) == 0:
        raise ValueError("No object found in the mask.")

    xmin, xmax = x.min(), x.max()
    ymin, ymax = y.min(), y.max()

    # Determine padding size based on object dimensions
    height, width = ymax - ymin, xmax - xmin
    pad_x = int(width * padding)
    pad_y = int(height * padding)

    # Crop a region around the object with padding
    crop_xmin = max(xmin - pad_x, 0)
    crop_ymin = max(ymin - pad_y, 0)
    crop_xmax = min(xmax + pad_x, img.shape[1])
    crop_ymax = min(ymax + pad_y, img.shape[0])

    # Crop the object and its surrounding background from the mask and image
    object_crop_mask = mask[crop_ymin:crop_ymax, crop_xmin:crop_xmax]
    object_crop_img = img[crop_ymin:crop_ymax, crop_xmin:crop_xmax]

    # Scale the cropped region (including background and object)
    new_height = int(object_crop_mask.shape[0] * scale_factor)
    new_width = int(object_crop_mask.shape[1] * scale_factor)
    scaled_object_crop_mask = cv2.resize(object_crop_mask, (new_width, new_height), interpolation=cv2.INTER_NEAREST)
    scaled_object_crop_img = cv2.resize(object_crop_img, (new_width, new_height), interpolation=cv2.INTER_LINEAR)

    # Calculate position to center the scaled object in the new mask
    center_x = (xmin + xmax) // 2
    center_y = (ymin + ymax) // 2
    new_xmin = max(center_x - new_width // 2, 0)
    new_ymin = max(center_y - new_height // 2, 0)
    new_xmax = min(new_xmin + new_width, img.shape[1])
    new_ymax = min(new_ymin + new_height, img.shape[0])

    # Create new mask and image with the scaled object and adjusted background
    new_mask = np.zeros_like(mask)
    new_img = img.copy()

    new_mask[new_ymin:new_ymax, new_xmin:new_xmax] = scaled_object_crop_mask[:new_ymax-new_ymin, :new_xmax-new_xmin]
    new_img[new_ymin:new_ymax, new_xmin:new_xmax] = scaled_object_crop_img[:new_ymax-new_ymin, :new_xmax-new_xmin]

    # Convert back to PIL images if needed
    #new_img = Image.fromarray(new_img)
    #new_mask = Image.fromarray(new_mask)

    return new_img, new_mask


from PIL import Image
import numpy as np
import cv2

from PIL import Image
import numpy as np
import cv2

def scale_image_and_keep_mask_centered(img, mask, scale_factor):
    """
    Scales the entire image and mask, ensuring that the mask's object remains within the view.
    
    Parameters:
    img (PIL.Image or numpy array): The original image.
    mask (PIL.Image or numpy array): The binary mask image where non-zero regions represent the object.
    scale_factor (float): Scaling factor (e.g., 2.0 for double size, 0.5 for half size).
    
    Returns:
    new_img (PIL.Image): The modified image with the scaled region.
    new_mask (PIL.Image): The modified mask with the scaled region.
    """
    # Convert PIL images to numpy arrays if necessary
    if isinstance(img, Image.Image):
        img = np.array(img)
    if isinstance(mask, Image.Image):
        mask = np.array(mask)
    
    # Get bounding box of the object in the mask
    y, x = np.where(mask > 0)  # Find all non-zero points
    if len(x) == 0 or len(y) == 0:
        raise ValueError("No object found in the mask.")

    xmin, xmax = x.min(), x.max()
    ymin, ymax = y.min(), y.max()

    # Calculate the center of the mask object
    center_x = (xmin + xmax) // 2
    center_y = (ymin + ymax) // 2

    # Scale the entire image and mask
    original_height, original_width = img.shape[:2]
    new_height = int(original_height * scale_factor)
    new_width = int(original_width * scale_factor)
    
    # Resize the image and mask
    scaled_img = cv2.resize(img, (new_width, new_height), interpolation=cv2.INTER_LINEAR)
    scaled_mask = cv2.resize(mask, (new_width, new_height), interpolation=cv2.INTER_NEAREST)
    
    # Calculate the offset to keep the mask centered in the view
    offset_x = max(center_x * scale_factor - original_width // 2, 0)
    offset_y = max(center_y * scale_factor - original_height // 2, 0)

    # Crop to original size to keep mask centered in the output
    crop_xmin = int(offset_x)
    crop_ymin = int(offset_y)
    crop_xmax = min(crop_xmin + original_width, new_width)
    crop_ymax = min(crop_ymin + original_height, new_height)

    cropped_img = scaled_img[crop_ymin:crop_ymax, crop_xmin:crop_xmax]
    cropped_mask = scaled_mask[crop_ymin:crop_ymax, crop_xmin:crop_xmax]

    # Convert back to PIL images if needed
    #new_img = Image.fromarray(cropped_img)
    #new_mask = Image.fromarray(cropped_mask)

    return cropped_img, cropped_mask


def scale_image_with_mask(img, mask, scale_factor, padding=0.25):
    """
    Scales a region of the image (including background and mask) around the object in the mask by a given factor.
    
    Parameters:
    img (PIL.Image or numpy array): The original image.
    mask (PIL.Image or numpy array): The binary mask image where non-zero regions represent the object.
    scale_factor (float): Scaling factor (e.g., 2.0 for double, 0.5 for half).
    padding (float): Fractional padding to include around the object during scaling. For example, 0.25 adds 25% padding.
    
    Returns:
    new_img (PIL.Image): The modified image with the scaled region.
    new_mask (PIL.Image): The modified mask with the scaled region.
    """
    # Convert PIL images to numpy arrays if necessary
    if isinstance(img, Image.Image):
        img = np.array(img)
    if isinstance(mask, Image.Image):
        mask = np.array(mask)
    
    # Get bounding box of the object in the mask
    y, x = np.where(mask > 0)  # Find all non-zero points
    if len(x) == 0 or len(y) == 0:
        raise ValueError("No object found in the mask.")

    xmin, xmax = x.min(), x.max()
    ymin, ymax = y.min(), y.max()

    # Determine padding size based on object dimensions
    height, width = ymax - ymin, xmax - xmin
    pad_x = int(width * padding)
    pad_y = int(height * padding)

    # Crop a region around the object with padding
    crop_xmin = max(xmin - pad_x, 0)
    crop_ymin = max(ymin - pad_y, 0)
    crop_xmax = min(xmax + pad_x, img.shape[1])
    crop_ymax = min(ymax + pad_y, img.shape[0])

    # Crop the region containing the object and background from the mask and image
    region_crop_mask = mask[crop_ymin:crop_ymax, crop_xmin:crop_xmax]
    region_crop_img = img[crop_ymin:crop_ymax, crop_xmin:crop_xmax]

    # Scale the cropped region (both mask and image)
    new_height = int(region_crop_mask.shape[0] * scale_factor)
    new_width = int(region_crop_mask.shape[1] * scale_factor)
    scaled_region_crop_mask = cv2.resize(region_crop_mask, (new_width, new_height), interpolation=cv2.INTER_NEAREST)
    scaled_region_crop_img = cv2.resize(region_crop_img, (new_width, new_height), interpolation=cv2.INTER_LINEAR)

    # Calculate position to center the scaled region in the new mask and image
    center_x = (crop_xmin + crop_xmax) // 2
    center_y = (crop_ymin + crop_ymax) // 2
    new_xmin = max(center_x - new_width // 2, 0)
    new_ymin = max(center_y - new_height // 2, 0)
    new_xmax = min(new_xmin + new_width, img.shape[1])
    new_ymax = min(new_ymin + new_height, img.shape[0])

    # Create new mask and image with the scaled region placed in the correct position
    new_mask = np.zeros_like(mask)
    new_img = img.copy()

    new_mask[new_ymin:new_ymax, new_xmin:new_xmax] = scaled_region_crop_mask[:new_ymax-new_ymin, :new_xmax-new_xmin]
    new_img[new_ymin:new_ymax, new_xmin:new_xmax] = scaled_region_crop_img[:new_ymax-new_ymin, :new_xmax-new_xmin]

    # Convert back to PIL images if needed
    #new_img = Image.fromarray(new_img)
    #new_mask = Image.fromarray(new_mask)

    return new_img, new_mask

#datapath /datasegswap
#inference_path /inference_xmem_ego_last/coco
#output /vis_piano
#--show_gt要加上
if __name__ == "__main__":

    #实验需改动
    root_path = "/data/work2-gcp-europe-west4-a/yuqian_fu/Ego/data_segswap/predictions/exo_query_test/92b2221b-ae92-44f0-bb31-e2d27cb736d6/aria01_214-1"
    file_names = natsorted(os.listdir(root_path))
    idxs = [int(f.split(".")[0]) for f in file_names]

    tmp = root_path.split("/")
    datapath = "/data/work2-gcp-europe-west4-a/yuqian_fu/Ego/data_segswap"
    take_id = tmp[-2]
    target_cam = tmp[-1]
    out_path = f"/data/work2-gcp-europe-west4-a/yuqian_fu/Ego/data_segswap/vis_psalm/exo_query_test/{take_id}/{target_cam}"
    os.makedirs(
        out_path, exist_ok=True
    )
    print(take_id, target_cam)

    #为了节省内存 实际上可以idx[:60]来可视化部分帧
    idxs = idxs[:2]
    for id in idxs:
        frame_idx = str(id)
        frame = cv2.imread(
            f"{datapath}/{take_id}/{target_cam}/{frame_idx}.jpg"
        )
        mask = Image.open(f"{root_path}/{frame_idx}.png")
        mask = np.array(mask)
        mask = cv2.resize(mask, (frame.shape[1], frame.shape[0]))

        try:
            mask = upsample_mask(mask, frame)
            out = blend_mask(frame, mask)
        except:
            breakpoint()

        cv2.imwrite(
            f"{out_path}/{frame_idx}.jpg",
            out,
        )


        #scale img: 2
        print('frame:', frame.shape, 'mask:', mask.shape)
        #img_new, mask_new = scale_mask_object(frame, mask, 0.5)
        #img_new, mask_new = scale_mask_object_with_background(frame, mask, 0.5)
        #img_new, mask_new = scale_image_with_mask(frame, mask, 0.5)
        img_new, mask_new = scale_image_and_keep_mask_centered(frame, mask, 0.25)
        print('img_new:', img_new.shape, 'mask_new:', mask_new.shape)
        out_new = blend_mask(img_new, mask_new)
        print('img saved at:', f"{out_path}/{frame_idx}_new.jpg")
        cv2.imwrite(
            f"{out_path}/{frame_idx}_new.jpg",
            out_new,
        )