# /*--------------------------------------------------------------------------------------------- # * Copyright (c) 2022-2023 STMicroelectronics. # * All rights reserved. # * # * This software is licensed under terms that can be found in the LICENSE file in # * the root directory of this software component. # * If no LICENSE file comes with this software, it is provided AS-IS. # *--------------------------------------------------------------------------------------------*/ import tensorflow as tf from common.data_augmentation import check_dataaug_argument, remap_pixel_values_range, apply_change_rate def _check_rectangle_erasing_args(fill_method, nrec, area, wh_ratio): """ Function checking if erasing parameters are valid. Potentially raises an error Args: fill_method (str): possible choices "uniform", "random", "mosaic" nrec: number of erased rectangles area: area of erased rectangles wh_ratio: erased rectangles form factor Returns: nrec, area, wh_ratio: tuple of min/max number of rectangles, area and width/height ratio """ if fill_method not in {"uniform", "random", "mosaic"}: raise ValueError("\nArgument `fill_method` of function `random_rectangle_erasing`: expecting " "'uniform', 'random' or 'mosaic'. Received {}".format(fill_method)) check_dataaug_argument(nrec, "nrec", function_name="random_rectangle_erasing", data_type=int) if not isinstance(nrec, (tuple, list)): nrec = (nrec, nrec) if nrec[0] < 0 or nrec[1] < 1: raise ValueError("\nArgument `nrec` of function `random_rectangle_erasing`: expecting " "an integer greater than 0 or a tuple of 2 integers (n1, n2) with n1 " "greater than or equal to 0. Received {}".format(nrec)) check_dataaug_argument(area, "area", function_name="random_rectangle_erasing", data_type=(int, float)) if not isinstance(area, (tuple, list)): area = (area, area) if area[0] < 0 or area[0] > 1 or area[1] < 0 or area[1] > 1: raise ValueError("\nArgument `area` of function `random_rectangle_erasing`: expecting a float " "or a tuple of 2 floats greater than 0.0 and less than 1.0. Received {}".format(area)) check_dataaug_argument(wh_ratio, "wh_ratio", function_name="random_rectangle_erasing", data_type=(int, float)) if not isinstance(wh_ratio, (tuple, list)): wh_ratio = (wh_ratio, wh_ratio) if wh_ratio[0] < 0 or wh_ratio[1] < 0: raise ValueError("\nArgument `wh_ratio` of function `random_rectangle_erasing`: expecting a float " "or a tuple of 2 floats greater than or equal to 0.0. Received {}".format(wh_ratio)) return nrec, area, wh_ratio def _parse_color_arg(color, fill_method=None, function_name=None): """ This function checks that argument `color` is set properly depending on the fill method. It returns an RGB color specification as a tensor with shape (3, 2). Each element is an integer in the interval [0, 255]. Args: color (tuple): tuple of 3 integers in the interval [0, 255] fill_method (str): possible choices "uniform", "random", "mosaic" function_name (str): augmentation function name Returns: (tf.Tensor) with RGB color specification """ if fill_method == "uniform": if color: message = f"\nArgument `color` of function `{function_name}`: expecting a " + \ f"tuple of 3 integers in the interval [0, 255]. Received {color}" if not isinstance(color, (tuple, list)) or len(color) != 3: raise ValueError(message) if type(color[0]) != int or type(color[1]) != int or type(color[2]) != int: raise ValueError(message) if color[0] < 0 or color[0] > 255 or color[1] < 0 or color[1] > 255 or color[2] < 0 or color[2] > 255: raise ValueError(message) color_spec = ((color[0], color[0]), (color[1], color[1]), (color[2], color[2])) else: # By default, rectangles are black. color_spec = ((0, 0), (0, 0), (0, 0)) else: # Fill method is either 'random' or 'mosaic'. if color: raise ValueError("\nFunction `{}`: argument `color` is only applicable to fill " "method 'uniform'. Received {}".format(function_name, color)) # All colors are allowed. color_spec = ((0, 255), (0, 255), (0, 255)) return tf.convert_to_tensor(color_spec, dtype=tf.int32) def _look_for_grayscale_rgb_images(images): """ This function looks for grayscale images that may be present in input RGB images. Their R, G and B planes are identical. We have to insert grayscale rectangles in these images instead of color rectangles, so we need to know where they are. Args: images (tf.Tensor): batch of input images Returns: (tf.Tensor) grayscale images indexes """ images_shape = tf.shape(images) batch_size = images_shape[0] width = images_shape[1] height = images_shape[2] r = tf.reshape(images[..., 0], [batch_size, width * height]) g = tf.reshape(images[..., 1], [batch_size, width * height]) b = tf.reshape(images[..., 2], [batch_size, width * height]) # Check if red, green and blue channels are the same rg = tf.cast(r != g, tf.int32) rb = tf.cast(r != b, tf.int32) s = tf.reduce_sum(rg, axis=-1) + tf.reduce_sum(rb, axis=-1) grayscale = tf.cast(s == 0, tf.int32) return grayscale def _generate_rectangle_mask(batch_size, image_size=None, num_planes=None, nrec_range=None, area_range=None, wh_ratio_range=None): """ This function first generates a rectangle in each plane. The coordinates (x, y) of the rectangle centers are chosen randomly. Their areas and width/height aspect ratios are sampled from the specified ranges. All the rectangles are clipped to the image size. Then, a number of rectangles is sampled for each image in the specified range. Some of the rectangles are suppressed to meet the required number of rectangles for each image. The rectangles are represented by a mask generated by the function that has the following shape: [batch_size, num_planes, image_width, image_height] Each element of the mask is a pixel of a plane. If the mask is set to 1, a rectangle is present at this location. If it is set to 0, no rectangle is present. Args: batch_size (int): number of images in a batch image_size (tuple): images dimensions (width, height) num_planes (int): number of images channels nrec_range (tuple): min/max number of erased rectangles area_range (tuple): min/max areas of erased rectangles wh_ratio_range (tuple): min/max width/height of erased rectangles Returns: (tf.Tensor): mask of erased rectangles """ width, height = image_size # Sample rectangle areas, aspect ratios and center coordinates of rectangles area = tf.random.uniform([batch_size, num_planes], minval=area_range[0], maxval=area_range[1], dtype=tf.float32) wh_ratio = tf.random.uniform([batch_size, num_planes], minval=wh_ratio_range[0], maxval=wh_ratio_range[1], dtype=tf.float32) x = tf.random.uniform([batch_size, num_planes], minval=0, maxval=width + 1, dtype=tf.int32) y = tf.random.uniform([batch_size, num_planes], minval=0, maxval=height + 1, dtype=tf.int32) # Calculate width and height of rectangles area = area * tf.cast(width, tf.float32) * tf.cast(height, tf.float32) w = tf.math.sqrt(wh_ratio * area) h = w / wh_ratio # Calculate corner coordinates of rectangles x = tf.cast(x, tf.float32) x1 = tf.cast(tf.round(x - w/2), tf.int32) x2 = tf.cast(tf.round(x + w/2), tf.int32) y = tf.cast(y, tf.float32) y1 = tf.cast(tf.round(y - h/2), tf.int32) y2 = tf.cast(tf.round(y + h/2), tf.int32) # Clip corner coordinates to the image size x1 = tf.math.maximum(x1, 0) y1 = tf.math.maximum(y1, 0) x2 = tf.math.minimum(x2, width) y2 = tf.math.minimum(y2, height) # Sample a number of rectangles for each image. Then, suppress # some of the rectangles so that the required number of rectangles # is met for each image. To eliminate a rectangle, we set its # 4 corner coordinates to 0. nrec = tf.random.uniform([batch_size, 1], minval=nrec_range[0], maxval=nrec_range[1] + 1, dtype=tf.int32) i = tf.range(batch_size * num_planes) % num_planes i = tf.reshape(i, [batch_size, num_planes]) selected_recs = tf.where(i < nrec, 1, 0) x1 = tf.reshape(x1 * selected_recs, [batch_size, num_planes, 1, 1]) y1 = tf.reshape(y1 * selected_recs, [batch_size, num_planes, 1, 1]) x2 = tf.reshape(x2 * selected_recs, [batch_size, num_planes, 1, 1]) y2 = tf.reshape(y2 * selected_recs, [batch_size, num_planes, 1, 1]) # Generate a mask that has 0's inside (x1, x2) # intervals and 1's outside i = tf.range(batch_size * height * num_planes) % height i = tf.reshape(i, [batch_size, num_planes, height]) i = tf.tile(i, [1, width, 1]) grid_x = tf.reshape(i, [batch_size, num_planes, width, height]) mask_x = tf.math.logical_and(grid_x >= x1, grid_x <= x2) # Generate a mask that has 0's inside (y1, y2) # intervals and 1's outside i = tf.range(batch_size * width * num_planes) % width i = tf.reshape(i, [batch_size, 1, num_planes, width]) i = tf.repeat(i, height) grid_y = tf.reshape(i, [batch_size, num_planes, width, height]) mask_y = tf.math.logical_and(grid_y >= y1, grid_y <= y2) mask = tf.math.logical_and(mask_x, mask_y) return tf.cast(mask, tf.int32) def _remove_rectangle_overlaps(rectangle_mask): """ This function removes any overlap between the rectangles that will be inserted in an image. This is needed to avoid the superimposition of colors in overlap areas. A number is assigned to each of the rectangles to be erased from an image. When several rectangles are present at the same pixel location, the rectangle with the largest number wins and remains intact. The other rectangles get carved and become more complex shapes such as L-shapes, disconnected smaller rectangles, etc. Args: rectangle_mask (tf.Tensor): mask of erased rectangles Returns: (tf.Tensor): filtered mask without overlaps """ mask_shape = tf.shape(rectangle_mask) batch_size = mask_shape[0] num_planes = mask_shape[1] width = mask_shape[2] height = mask_shape[3] # Assign a number from 1 to nrec to each rectangle n = tf.range(num_planes) + 1 n = tf.repeat(n, width * height) n = tf.reshape(n, [num_planes, width, height]) n = tf.tile(n, [batch_size, 1, 1]) n = tf.reshape(n, [batch_size, num_planes, width, height]) numbered_recs = n * tf.cast(rectangle_mask, dtype=tf.int32) winner_numbers = tf.math.reduce_max(numbered_recs, axis=1) winner_numbers = tf.expand_dims(winner_numbers, axis=1) mask = tf.where(numbered_recs == winner_numbers, winner_numbers, 0) # The mask has 1's inside rectangle areas. return tf.where(mask > 0, 1, 0) def _generate_color_planes(batch_size, image_size=None, num_planes=None, fill_method=None, color=None): """ This function creates planes with uniform RGB colors. The rectangles to insert in the images will be cut from these planes to obtain colored rectangles. The shape of the tensor generated by the function is: [batch_size, num_planes, width, height, 3] The color of each plane depends on the fill method and the color specification. If the fill method is: - "random", the color of each plane is randomly chosen. All RGB colors are usable. All the rectangles will have different colors. - "uniform", all the planes have the same RGB color that is specified by the `color` argument (a tuple of 3 integers in the [0, 255] interval). - "mosaic", the planes are filled with random RGB pixels. Args: batch_size (int): number of images in a batch image_size (tuple): images dimensions (width, height) num_planes (int): number of planes fill_method (str): possible choices "uniform", "random", "mosaic" color (tuple): color ranges for each image channels Returns: (tf.Tensor): planes with uniform RGB colors """ width, height = image_size if fill_method == "uniform" or fill_method == "random": # If the fill method is "uniform", color[i, 0] and color[i, 1] have the # same value. If the fill method is "random", they have different values. r = tf.random.uniform([batch_size * num_planes], minval=color[0, 0], maxval=color[0, 1] + 1, dtype=tf.int32) r = tf.reshape(r, [batch_size, num_planes]) r = tf.repeat(r, width * height) g = tf.random.uniform([batch_size * num_planes], minval=color[1, 0], maxval=color[1, 1] + 1, dtype=tf.int32) g = tf.reshape(g, [batch_size, num_planes]) g = tf.repeat(g, width * height) b = tf.random.uniform([batch_size * num_planes], minval=color[2, 0], maxval=color[2, 1] + 1, dtype=tf.int32) b = tf.reshape(b, [batch_size, num_planes]) b = tf.repeat(b, width * height) else: # The fill method is "mosaic". pixels = batch_size * num_planes * width * height r = tf.random.uniform([pixels], minval=color[0, 0], maxval=color[0, 1] + 1, dtype=tf.int32) g = tf.random.uniform([pixels], minval=color[1, 0], maxval=color[1, 1] + 1, dtype=tf.int32) b = tf.random.uniform([pixels], minval=color[2, 0], maxval=color[2, 1] + 1, dtype=tf.int32) r = tf.reshape(r, [batch_size, num_planes, width, height]) g = tf.reshape(g, [batch_size, num_planes, width, height]) b = tf.reshape(b, [batch_size, num_planes, width, height]) x = tf.stack([r, g, b], axis=-1) color_planes = tf.reshape(x, [batch_size, num_planes, width, height, 3]) return color_planes def random_rectangle_erasing( images, nrec=1, area=(0.05, 0.4), wh_ratio=(0.3, 2.0), fill_method="random", color=None, pixels_range=(0.0, 1.0), change_rate=1.0, mode="image"): """ This function randomly erases a number of rectangular areas from input images and fills the voids with color. The function has two modes: - image mode: different sets of random rectangles are erased from the different images of the batch. - batch mode: the same random set of images is erased from all the images of the batch. The image mode creates more image diversity, potentially leading to better training results, but run times are longer than in the batch mode. By default, if no arguments are specified, one random black rectangle is erased from each image. All the rectangles are different (image mode). Args: images: Input RGB or grayscale images with shape [batch_size, width, height, channels]. nrec: An integer or a tuple of 2 integers, specifies the range of values the numbers of rectangles to erase are sampled from (one number of rectangles per image). If a scalar value v is used, the number of rectangles is equal to v for all the images. area: A float or a tuple of 2 floats, specifies the range of values the areas of rectangles are sampled from (one area per rectangle). If a scalar value v is used, the area of all the rectangles in all the images is equal to v. Area values are fractions of the image area. Therefore, they must be in the interval [0, 1]. wh_ratio: A float or a tuple of 2 floats, specifies the range of values the width/height aspect ratio of rectangles are sampled from (one ratio per rectangle). If a scalar value v is used, the aspect ratio of all the rectangles in all the images is equal to v. fill_method: A string, one of {"uniform", "random" or "mosaic"}, specifies the method to use to fill with color the rectangles erased from the images. If the fill method is: "uniform": All the rectangles in all the images are filled with the same color specified using the `color` argument. If no color is specified, all the rectangles are black. "random": Each rectangle is filled with a randomly chosen RGB color if the input images are RGB images or a randomly chosen shade of gray if the input images are grayscale (all the rectangles will have different colors). "mosaic": Each rectangle is filled with random pixels. color: A tuple of 3 integers in the interval [0, 255] if the images are RGB images or an integer in the interval [0, 255] if the images are grayscale images, specifies the color to use to fill the rectangles erased from the images. This argument is only applicable to the 'uniform' fill method. change_rate: A float in the interval [0, 1], the number of changed images versus the total number of input images average ratio. For example, if `change_rate` is set to 0.25, 25% of the input images will get changed on average (75% won't get changed). If it is set to 0.0, no images are changed. If it is set to 1.0, all the images are changed. pixels_range: A tuple of 2 integers or floats, specifies the range of pixel values in the input images and output images. Any range is supported. It generally is either [0, 255], [0, 1] or [-1, 1]. mode: Either "image" or "batch". If set to "image", different sets of rectangles are erased from the different images of the batch. If set to "batch", the same set of rectangles is erased from all the images of the batch. The "image" mode creates more image diversity, potentially leading to better training results, but run times are longer than in the "batch" mode. Returns: The images with erased rectangles. """ images_shape = tf.shape(images) batch_size = images_shape[0] width = images_shape[1] height = images_shape[2] pixels_dtype = images.dtype images = remap_pixel_values_range(images, pixels_range, (0, 255), dtype=tf.int32) nrec, area, wh_ratio = _check_rectangle_erasing_args(fill_method, nrec, area, wh_ratio) color = _parse_color_arg(color, fill_method=fill_method, function_name="random_rectangle_erasing") if mode not in {"image", "batch"}: raise ValueError("\nArgument `mode` of function `random_rectangle_erasing`: " "expecting 'image' or 'batch'. Received {}".format(mode)) grayscale = _look_for_grayscale_rgb_images(images) # We call "planes" areas of the same size as the input images that # contain the rectangles to erase from the images. A set of planes # is associated to each image. Each plane either contains one # rectangle to erase from the image or no rectangle (empty plane). # The number of planes associated to an image is equal to the # maximum number of rectangles that may be erased from the image. num_planes = nrec[1] rectangle_mask = _generate_rectangle_mask( batch_size=batch_size if mode == "image" else 1, image_size=[width, height], num_planes=num_planes, nrec_range=nrec, area_range=area, wh_ratio_range=wh_ratio) if nrec != (1, 1): rectangle_mask = _remove_rectangle_overlaps(rectangle_mask) # Color the rectangles and compute the union of the # planes to obtain images of all the rectangles. color_planes = _generate_color_planes( batch_size=batch_size if mode == "image" else 1, image_size=[width, height], num_planes=num_planes, fill_method=fill_method, color=color) colored_rectangles = tf.expand_dims(rectangle_mask, axis=-1) * color_planes rectangle_images = tf.math.reduce_sum(colored_rectangles, axis=1) if mode == "batch": # We have only one image of rectangles. Duplicate it to have a batch. rectangle_images = tf.tile(rectangle_images, [batch_size, 1, 1, 1]) rectangle_images = tf.reshape(rectangle_images, [batch_size, width, height, 3]) # If the image is grayscale, the rectangles must also be grayscale. gray_rectangle_images = tf.stack([rectangle_images[..., 0], rectangle_images[..., 0], rectangle_images[..., 0]], axis=-1) grayscale = tf.reshape(grayscale, [batch_size,1, 1, 1]) rectangle_images = tf.where(grayscale == 1, gray_rectangle_images, rectangle_images) # Insert the rectangles in the input images rectangle_mask = tf.expand_dims(rectangle_mask, axis=-1) mask = tf.math.reduce_sum(rectangle_mask, axis=1) images_aug = tf.where(mask == 0, images, rectangle_images) outputs = apply_change_rate(images, images_aug, change_rate) return remap_pixel_values_range(outputs, (0, 255), pixels_range, dtype=pixels_dtype)