Module brevettiai.data.image.image_augmenter

Expand source code
import logging
import tensorflow as tf
import tensorflow_addons as tfa
import numpy as np
from brevettiai.data.image import ImageKeys
from brevettiai.interfaces import vue_schema_utils as vue
from functools import partial

log = logging.getLogger(__name__)


@tf.function
def _gaussian_kernel(kernel_size, sigma, dtype):
    x = tf.range(-kernel_size // 2, kernel_size // 2 + 1, dtype=dtype)
    g = tf.math.exp(-(tf.pow(x, 2) / (2 * tf.pow(tf.cast(sigma, dtype), 2))))
    g_kernel = tf.tensordot(g, g, axes=0)
    return g_kernel / tf.reduce_sum(g_kernel)


@tf.function
def gaussian_blur(x, sigma):
    kernel_size = tf.cast(tf.math.ceil(sigma * 2.5), tf.int32) * 2 + 1
    gaussian_blur_kernel = tf.tile(_gaussian_kernel(kernel_size, sigma, x.dtype)[..., None, None],
                                   (1, 1, tf.shape(x)[-1], 1))
    x = tf.nn.depthwise_conv2d(x, gaussian_blur_kernel, [1, 1, 1, 1], 'SAME')
    return x


@tf.function
def _avg_kernel(kernel_size, dtype):
    g_kernel = tf.ones(kernel_size, dtype=dtype)
    return g_kernel / tf.reduce_sum(g_kernel)


@tf.function
def affine(o1, o2, r1, r2, sc, a, sh1, sh2, t1, t2):
    """
    Create N affine transform matrices from output to input (Nx3x3)
    for use with tfa.transform
    :param o1: Origin 1 (shape / 2 for center)
    :param o2: Origin 2
    :param r1: Reference scale 1 (1 or -1 for flip)
    :param r2: Reference scale 1
    :param sc: Augmented scale
    :param a: Rotation in radians
    :param sh1: Shear 1
    :param sh2: Shear 2
    :param t1: Translate 1
    :param t2: Translate 2
    :return: Inverse transformation matrix
    """
    _0 = tf.zeros_like(o1)
    _1 = tf.ones_like(o1)
    T1 = tf.transpose(tf.convert_to_tensor([[_1, _0, -(o2 - 1)],
                                            [_0, _1, -(o1 - 1)],
                                            [_0, _0, _1]], dtype=tf.float32), [2, 0, 1])
    T2 = tf.transpose(tf.convert_to_tensor([[_1, _0, (o2 - 1)],
                                            [_0, _1, (o1 - 1)],
                                            [_0, _0, _1]], dtype=tf.float32), [2, 0, 1])
    ref = tf.transpose(tf.convert_to_tensor([[r2, _0, _0],
                                             [_0, r1, _0],
                                             [_0, _0, _1]], dtype=tf.float32), [2, 0, 1])
    scale = tf.transpose(tf.convert_to_tensor([[1 + sc, _0, _0],
                                               [_0, 1 + sc, _0],
                                               [_0, _0, _1]], dtype=tf.float32), [2, 0, 1])
    rot = tf.transpose(tf.convert_to_tensor([[tf.cos(a), -tf.sin(a), _0],
                                             [tf.sin(a), tf.cos(a), _0],
                                             [_0, _0, _1]], dtype=tf.float32), [2, 0, 1])
    she = tf.transpose(tf.convert_to_tensor([[_1, sh2, _0],
                                             [sh1, _1, _0],
                                             [_0, _0, _1]], dtype=tf.float32), [2, 0, 1])
    tra = tf.transpose(tf.convert_to_tensor([[_1, _0, t2],
                                             [_0, _1, t1],
                                             [_0, _0, _1]], dtype=tf.float32), [2, 0, 1])
    return T2 @ tra @ she @ ref @ scale @ rot @ T1


class RandomTransformer(vue.VueSettingsModule):
    def __init__(self, chance: float = 0.5, flip_up_down: bool = True, flip_left_right: bool = True,
                 scale: float = 0.2, rotate_chance: float = 0.5, rotate: float = 90,
                 translate_horizontal: float = 0.1, translate_vertical: float = 0.1, shear: float = 0.04,
                 interpolation: str = "bilinear"):
        """
        Build random transformation matrices for batch of images
        :param shape:
        :param chance:
        :param flip:
        :param scale:
        :param rotate:
        :param translate:
        :param shear:
        :param interpolation: Resampling interpolation method
        :return:
        """

        self.chance = chance
        self.flip_up_down = flip_up_down
        self.flip_left_right = flip_left_right
        self.scale = scale
        self.rotate_chance = rotate_chance
        self.rotate = rotate
        self.translate_horizontal = translate_horizontal
        self.translate_vertical = translate_vertical
        self.shear = shear
        self.interpolation = interpolation
        self.fill_seg_value = 0.0

    def set_fill_seg_value(self, fill_value):
        self.fill_seg_value = fill_value

    def transform_images(self, x, A, interpolation=None, fill_seg=False):
        """
        :param x: 4D image tensor (batch_size x height x width x channels)
        :param A: 3D stack of affine (batch_size x 3 x 3) type is always float32
        """

        tr = tfa.image.transform_ops.matrices_to_flat_transforms(A)
        x = tfa.image.transform(x, tr, interpolation or self.interpolation)

        if fill_seg:
            mask = tfa.image.transform(tf.zeros_like(x[..., :1]), tr, interpolation or self.interpolation,
                                       fill_value=0.0)
            x += mask * self.fill_seg_value

        return x

    def __call__(self, shape, seed):
        sh = shape
        origin = tf.cast(sh[1:3], np.float32)[None] / 2 * tf.ones((sh[0], 1))

        flip = (self.flip_up_down, self.flip_left_right)

        mask_rotate = tf.cast(tf.random.uniform(sh[:1], seed=seed) < self.rotate_chance, tf.float32)

        rotate = self.rotate
        try:
            iter(rotate)
        except TypeError:
            rotate = [-rotate, rotate]
        rotate = np.array(rotate) * np.pi / 180
        rotate = tf.random.uniform(sh[:1], *rotate, seed=seed + 1) * mask_rotate

        mask_flip = tf.cast(tf.random.uniform(sh[:1], seed=seed) < self.chance, tf.float32)
        if flip[0]:
            flip1 = tf.sign(tf.random.uniform(sh[:1], -1, 1, seed=seed + 2)) * mask_flip + (1-mask_flip)
        else:
            flip1 = tf.ones_like(origin[:, 0])
        if flip[1]:
            flip2 = tf.sign(tf.random.uniform(sh[:1], -1, 1, seed=seed + 3)) * mask_flip + (1-mask_flip)
        else:
            flip2 = tf.ones_like(origin[:, 0])

        mask_scale_shear = tf.cast(tf.random.uniform(sh[:1], seed=seed + 4) < self.chance, tf.float32)
        scale = self.scale
        try:
            iter(scale)
        except TypeError:
            scale = [-scale, scale]
        scale = tf.random.uniform(sh[:1], *scale, seed=seed + 5) * mask_scale_shear
        shear = self.shear
        try:
            iter(shear)
        except TypeError:
            shear = [-shear, shear]
        shear1 = tf.random.uniform(sh[:1], *shear, seed=seed + 8) * mask_scale_shear
        shear2 = tf.random.uniform(sh[:1], *shear, seed=seed + 9) * mask_scale_shear

        mask_translate = tf.cast(tf.random.uniform(sh[:1], seed=seed) < self.chance, tf.float32)
        translate = [[-self.translate_vertical, self.translate_vertical],
                     [-self.translate_horizontal, self.translate_horizontal]]

        translate = tf.convert_to_tensor(translate) * tf.cast(sh[1:3], tf.float32)[:, None]
        translate1 = tf.random.uniform(sh[:1], translate[0, 0], translate[0, 1], seed=seed + 6) * mask_translate
        translate2 = tf.random.uniform(sh[:1], translate[1, 0], translate[1, 1], seed=seed + 7) * mask_translate

        A = affine(origin[:, 0], origin[:, 1], flip1, flip2, scale, rotate, shear1, shear2, translate1, translate2)
        return A


class ImageDeformation(vue.VueSettingsModule):
    def __init__(self, alpha: float = 0.0, sigma: float = 0.5, chance: float = 0.5):
        self.alpha = alpha
        self.sigma = sigma
        self.chance = chance

    def __call__(self, shape, seed):
        deformations = tf.random.uniform((shape[0], shape[1], shape[2], 2), seed=seed+114)
        probabilities = tf.random.uniform((shape[0], ), seed=seed + 113)
        return deformations, probabilities

    def deform_image(self, inputs, interpolation="bilinear"):
        x, dxy, do_deform = inputs
        if do_deform < self.chance:
            shape = tf.shape(x)
            dxy = gaussian_blur(dxy[None] * 2 - 1, self.sigma) * self.alpha
            if interpolation and interpolation == "nearest":
                dxy = tf.round(dxy)

            gx, gy = tf.meshgrid(tf.range(shape[1], dtype=tf.float32), tf.range(shape[0], dtype=tf.float32), indexing="xy")
            grid = tf.stack([gx + dxy[..., 0],
                             gy + dxy[..., 1]], -1)
            return tfa.image.resampler(x[None], grid)[0]
        else:
            return x

    def apply(self, x, deformations, probabilities, interpolation="bilinear"):
        """Elastic deformation of images as described in [Simard2003]_ (with modifications).
        .. [Simard2003] Simard, Steinkraus and Platt, "Best Practices for
             Convolutional Neural Networks applied to Visual Document Analysis", in
             Proc. of the International Conference on Document Analysis and
             Recognition, 2003.

         Based on https://gist.github.com/erniejunior/601cdf56d2b424757de5
        """

        deformation = tf.map_fn(fn=partial(self.deform_image, interpolation=interpolation),
                                elems=(x, deformations,probabilities), dtype=x.dtype)
        return deformation


class ImageSaltAndPepper(vue.VueSettingsModule):
    def __init__(self, fraction: float = 0.0002, value_range: tuple = None, scale: int = 1, chance: float = 0.5):
        """
        :param brightness: Embossing in a random direction with a scale chosen in the given range
        :param contrast: Size of mean filtering kernel
        :param hue:
        :param saturation:
        :param stddev:
        :param chance: The chance of the individual step to be applied
        """
        self.fraction = fraction
        self.value_range = value_range or (0.0, 1.0)
        self.scale = scale
        assert scale in [1, 3, 5]
        self.chance = chance

        self.fractions = (self.fraction / 2, 1.0-self.fraction / 2)

    def apply_noise(self, inputs):
        x, salt_pepper, prob_salt_pepper = inputs
        if prob_salt_pepper < self.chance:
            if self.scale == 5:
                sp = tf.cast(-tf.nn.max_pool2d(-salt_pepper[None], (1, 5, 5, 1), (1, 1, 1, 1), "SAME") < self.fractions[0], tf.float32)[0] * \
                     self.value_range[0] + \
                     tf.cast(tf.nn.max_pool2d(salt_pepper[None], (1, 5, 5, 1), (1, 1 ,1, 1), "SAME") > self.fractions[1], tf.float32)[0] * \
                     self.value_range[1]
            elif self.scale == 3:
                sp = tf.cast(-tf.nn.max_pool2d(-salt_pepper[None], (1, 3, 3, 1), (1, 1, 1, 1), "SAME") < self.fractions[0], tf.float32)[0] * \
                     self.value_range[0] + \
                     tf.cast(tf.nn.max_pool2d(salt_pepper[None], (1, 3, 3, 1), (1, 1 ,1, 1), "SAME") > self.fractions[1], tf.float32)[0] * \
                     self.value_range[1]
            elif self.scale == 1:
                sp = tf.cast(salt_pepper < self.fractions[0], tf.float32) * self.value_range[0] + \
                     tf.cast(salt_pepper > self.fractions[1], tf.float32) * self.value_range[1]
            return sp
        else:
            return tf.zeros_like(salt_pepper)

    def __call__(self, x, seed):
        sh = tf.shape(x)

        salt_pepper = tf.random.uniform(shape=sh[:3], minval=0.0, maxval=1.0, dtype=tf.float32, seed=seed+111)[..., None]

        prob_salt_pepper = tf.random.uniform((sh[0], ), seed=seed+112)
        return tf.map_fn(self.apply_noise, [x, salt_pepper, prob_salt_pepper], dtype=tf.float32)


class ImageNoise(vue.VueSettingsModule):
    def __init__(self, brightness: float = 0.25, contrast: tuple = (0.5, 1.25),
                 hue: float = 0.05, saturation: tuple = (0, 2.0), stddev: float = 0.01, chance: float = 0.5):
        """
        :param brightness: Embossing in a random direction with a scale chosen in the given range
        :param contrast: Size of mean filtering kernel
        :param hue:
        :param saturation:
        :param stddev:
        :param chance: The chance of the individual step to be applied
        """
        self.brightness = brightness
        self.contrast = contrast or (0.5, 1.25)
        self.hue = hue
        self.saturation = saturation or (0, 2.0)
        self.stddev = stddev
        self.chance = chance

    def conditional_noise(self, inputs):
        x, hue, saturation, prob_hue_sat, contrast, prob_contrast, brightness, prob_brightness, noise, prob_noise = inputs
        sh = tf.shape(x)

        if sh[2] == 3 and prob_hue_sat < self.chance:
            x = tf.image.adjust_saturation(x, saturation)
            x = tf.image.adjust_hue(x, hue)

        if prob_contrast < self.chance:
            x = tf.image.adjust_contrast(x, contrast)

        if prob_brightness < self.chance:
            x = tf.image.adjust_brightness(x, brightness)

        if prob_noise < self.chance:
            x = tf.add(x, noise)
        return x

    def __call__(self, x, seed):
        sh = tf.shape(x)

        hue = tf.random.uniform((sh[0], ), -self.hue, self.hue, seed=seed + 100)
        saturation = tf.random.uniform((sh[0], ), self.saturation[0], self.saturation[1], seed=seed+101)
        prob_hue_sat = tf.random.uniform((sh[0],), seed=seed+102)

        contrast = tf.random.uniform((sh[0], ), self.contrast[0], self.contrast[1], seed=seed+103)
        prob_contrast = tf.random.uniform((sh[0],), seed=seed+104)

        brightness = tf.random.uniform((sh[0], ), -self.brightness, self.brightness, seed=seed+105)
        prob_brightness = tf.random.uniform((sh[0],), seed=seed+106)

        noise = tf.random.normal(shape=tf.shape(x), mean=0, stddev=self.stddev,
                                 dtype=tf.float32, seed=seed+107)
        prob_noise = tf.random.uniform((sh[0],), seed=seed+108)

        return tf.map_fn(self.conditional_noise, [x,
                                                  hue, saturation, prob_hue_sat,
                                                  contrast, prob_contrast,
                                                  brightness, prob_brightness,
                                                  noise, prob_noise], dtype=tf.float32)


class ImageFiltering(vue.VueSettingsModule):
    def __init__(self, emboss_strength: tuple = None, avg_blur: tuple = (3, 3), gaussian_blur_sigma: float = 0.5,
                 chance: float = 0.5):
        """
        :param emboss_strength: Embossing in a random direction with a scale chosen in the given range
        :param avg_blur: Size of mean filtering kernel
        :param gaussian_blur_sigma:
        :param chance: The chance of the individual step to be applied
        """
        self.emboss_strength = emboss_strength or (0, 0.25)
        self.avg_blur = avg_blur or (3, 3)
        self.gaussian_blur_sigma = gaussian_blur_sigma
        self.chance = chance

    def conditional_filter(self, inputs):
        x, probability, emboss_xy = inputs
        if probability < self.chance / 3:
            # Emboss
            dxy = tf.image.sobel_edges(x[None])[0]
            emboss = tf.reduce_sum(emboss_xy * dxy, axis=-1)
            return x + emboss
        elif probability < 2 * self.chance / 3:
            # Gaussian blur
            return gaussian_blur(x[None], self.gaussian_blur_sigma)[0]
        elif probability < 3 * self.chance / 3:
            # Average blur
            avg_blur_kernel =  tf.tile(_avg_kernel(self.avg_blur, x.dtype)[..., None, None], (1, 1, tf.shape(x)[-1], 1))
            return tf.nn.depthwise_conv2d(x[None], avg_blur_kernel, [1, 1, 1, 1], 'SAME')[0]
        else:
            # No filtering
            return x

    def __call__(self, x, seed):
        sh = tf.shape(x)
        probabilities = tf.random.uniform((sh[0],), seed=seed+102)

        emboss_sign = tf.sign(tf.random.uniform((sh[0], 1, 1, sh[3], 2), -1, 1, seed=seed + 103))
        emboss_xy = emboss_sign * tf.random.uniform((sh[0], 1, 1, sh[3], 2), self.emboss_strength[0],
                                                    self.emboss_strength[1], seed=seed + 104)
        return tf.map_fn(self.conditional_filter, [x, probabilities, emboss_xy], dtype=tf.float32)


class ViewGlimpseFromBBox(vue.VueSettingsModule):
    def __init__(self, bbox_key=None,
                 target_shape: tuple = None, zoom_factor: int = None):
        self.bbox_key = bbox_key or ImageKeys.BOUNDING_BOX
        self.target_shape = target_shape

        # Ensure zoom_factor is not 0
        self.zoom_factor = None if zoom_factor == 0 else zoom_factor

        if self.zoom_factor:
            assert np.log2(self.zoom_factor).is_integer(), \
                f"zoom_exp_factor '{self.zoom_factor}' must be a power of 2"

    @property
    def bbox_shape(self):
        if self.zoom_factor: # not zero or None
            return tuple(int(x * self.zoom_factor) for x in self.target_shape)
        else:
            return self.target_shape

    def __call__(self, x, seed):
        # Extract info
        img_size = tf.cast(x[ImageKeys.SIZE], tf.int32)
        views = tf.cast(x[self.bbox_key], tf.int32)

        # Choose glimpse in bbox
        offset_scales = views[:, 2:] - views[:, :2] + self.bbox_shape
        offsets = tf.random.uniform(tf.shape(offset_scales), seed=seed + 201) * tf.cast(offset_scales, tf.float32)
        view_center = views[:, :2] + tf.cast(tf.round(offsets), tf.int32)
        view_origin = view_center - self.bbox_shape

        # Clip bbox to boundaries of image
        max_idx = img_size - self.bbox_shape
        min_idx = tf.zeros_like(max_idx, dtype=tf.int32)
        view_origin = tf.clip_by_value(view_origin, min_idx, max_idx)

        # Update bbox
        views = tf.concat([view_origin, view_origin + tf.constant(self.bbox_shape)[None]], axis=1)
        x[self.bbox_key] = views

        # Add zoom to dataset
        if self.zoom_factor is not None:
            # Create tensor of shape (batch_size,) with value self.zoom_factor
            zef = tf.constant(self.zoom_factor, dtype=tf.float32, shape=(1,))
            x[ImageKeys.ZOOM] = tf.broadcast_to(zef, tf.shape(x[self.bbox_key])[:1])
        return x


class ViewGlimpseFromPoints(vue.VueSettingsModule):
    def __init__(self, bbox_key=None, target_shape: tuple = None, zoom_factor: int = None, overlap: int = 0.8):
        self.bbox_key = bbox_key or ImageKeys.BOUNDING_BOX
        self.target_shape = target_shape

        # Ensure zoom_factor is not 0
        self.zoom_factor = None if zoom_factor == 0 else zoom_factor

        self.overlap = overlap

        if self.zoom_factor and not np.log2(self.zoom_factor).is_integer():
            log.warning(f"zoom_exp_factor '{self.zoom_factor}' must be a power of 2 for tiled images to work")

    @property
    def bbox_shape(self):
        if self.zoom_factor: # not zero or None
            return tuple(int(x * self.zoom_factor) for x in self.target_shape)
        else:
            return self.target_shape

    def __call__(self, x, seed):
        inside_points = x[ImageKeys.INSIDE_POINTS]
        sh = tf.shape(inside_points)

        if ImageKeys.BBOX_SIZE_ADJUST in x:
            bbox_adjust = tf.cast(x[ImageKeys.BBOX_SIZE_ADJUST], tf.float32)
            adjusted_bbox_shape = bbox_adjust[:, None] * self.bbox_shape
            x[ImageKeys.ZOOM] = tf.round(adjusted_bbox_shape)[:, 0] / self.bbox_shape[0]
        else:
            adjusted_bbox_shape = tf.constant(self.bbox_shape, dtype=tf.float32)[None]

        # Choose target point
        choice = tf.random.uniform(shape=(sh[0],), maxval=sh[1], dtype=tf.int32, seed=seed + 202)
        target_points = tf.gather_nd(inside_points, indices=tf.stack([tf.range(sh[0]), choice], axis=-1))

        # Adjust bbox
        half_size = tf.cast(tf.round(adjusted_bbox_shape / 2), tf.int32)
        overlap_px = tf.cast(half_size, tf.float32) * self.overlap
        offset = tf.random.uniform(tf.shape(target_points), minval=-1, maxval=1, seed=seed + 201) * overlap_px
        view_origin = target_points - half_size + tf.cast(offset, tf.int32)

        # Clip bbox to boundaries of image
        img_size = tf.cast(x[ImageKeys.SIZE], tf.int32)
        max_idx = img_size - self.bbox_shape
        min_idx = tf.zeros_like(max_idx, dtype=tf.int32)
        view_origin = tf.clip_by_value(view_origin, min_idx, max_idx)

        # Update bbox
        views = tf.concat([view_origin, view_origin + tf.cast(tf.round(adjusted_bbox_shape), tf.int32)], axis=1)
        x[self.bbox_key] = views

        # Add zoom to dataset
        if self.zoom_factor is not None:
            # Create tensor of shape (batch_size,) with value self.zoom_factor
            zef = tf.constant(self.zoom_factor, dtype=tf.float32, shape=(1,))
            zef = tf.broadcast_to(zef, tf.shape(x[self.bbox_key])[:1])
            x[ImageKeys.ZOOM] = zef * x[ImageKeys.ZOOM] if ImageKeys.ZOOM in x else zef

        return x


class ImageAugmenter(vue.VueSettingsModule):
    def __init__(self, image_keys=None, label_keys=None,
                 random_transformer: RandomTransformer = RandomTransformer(),
                 image_noise: ImageNoise = ImageNoise(),
                 image_filter: ImageFiltering = ImageFiltering(),
                 image_deformation: ImageDeformation = None):
        """
        Image augmentation class, for use with tensorflow and criterion datasets
        The call method expects a tensor dict with image and label keys for transformation.
        Alternatively, the augmenter may be used by calling transform_images directly

        :param image_keys: labels of images to perform augmentation on
        :param label_keys: labels of annotations to perform augmentation on
        :param random_transformer: A random affine transformation object with RandomTransformer interfaces
        :param image_noise: An image noise generation object with ImageNoise interfaces
        :param image_filter: An image filter noise generation object with ImageFiltering interfaces
        :param image_deformation: A local random image deformation object with ImageDeformation interfaces

        """
        self.image_keys = image_keys or ["img"]
        self.label_keys = label_keys or ["segmentation", "annotation"]
        self.random_transformer = random_transformer
        self.image_noise = image_noise
        self.image_filter = image_filter
        self.image_deformation = image_deformation
        pass

    def __call__(self, x, seed, *args, **kwargs):
        A = deformations = probabilities = None

        if self.random_transformer is not None:
            A = self.random_transformer(tf.shape(x[self.image_keys[0]]), seed=seed)
        if self.image_deformation is not None:
            deformations, probabilities = self.image_deformation(tf.shape(x[self.image_keys[0]]), seed)

        for key in self.image_keys:
            if key in x:
                if self.random_transformer is not None:
                    x[key] = self.random_transformer.transform_images(x[key], A)
                if self.image_deformation is not None:
                    x[key] = self.image_deformation.apply(x[key], deformations, probabilities)
                if self.image_filter is not None:
                    x[key] = self.image_filter(x[key], seed=seed)
                if self.image_noise is not None:
                    x[key] = self.image_noise(x[key], seed=seed)

        for key in self.label_keys:
            if key in x:
                if self.random_transformer is not None:
                    x[key] = self.random_transformer.transform_images(tf.cast(x[key], tf.float32), A,
                                                                      interpolation="nearest", fill_seg=True)
                if self.image_deformation is not None:
                    x[key] = self.image_deformation.apply(tf.cast(x[key], tf.float32), deformations, probabilities, interpolation="nearest")
        return x

    @classmethod
    def to_schema(cls, builder, name, ptype, default, **kwargs):

        if name in {"image_keys", "label_keys"}:
            return
        else:
            return super().to_schema(builder=builder, name=name, ptype=ptype, default=default, **kwargs)



# DEPRECATED
def get_transform_schema(ns):
    return [
        vue.number_input("Vertical translation range in percent", ns + "translate_vertical", default=0.1, required=False, visible=False),
        vue.number_input("Horizontal translation range in percent", ns + "translate_horizontal", default=0.1, required=False, visible=False),
        vue.number_input("Maximum scale offset", ns + "scale", default=0.2, required=False, visible=False),
        vue.number_input("Maximum relative shear", ns + "shear", default=0.04, required=False, visible=False),
        vue.number_input("Maximum rotation angle", ns + "rotate", default=90, required=False, visible=False),
        vue.checkbox("Flip up-down", ns + "flip_up_down", default=True, required=False, visible=False),
        vue.checkbox("Flip left-right", ns + "flip_left_right", default=True, required=False, visible=False),
        vue.number_input("Chance of augmentation steps", ns + "chance", default=0.5, required=False, visible=False)
    ]


# DEPRECATED
def get_noise_schema(ns):
    return [
        vue.number_input("Maximum brightness delta", ns + "brightness", default=0.25, required=False, visible=False),
        vue.text_input("Emboss alpha range", ns + "emboss_alpha",
                       required=False, default="[0.0, 0.1]", json=True),
        vue.text_input("Emboss strength range", ns + "emboss_alpha",
                       required=False, default="[0.75, 1.25]", json=True),
        vue.text_input("Image saturation range", ns + "saturation",
                       required=False, default="[0, 2.0]", json=True),
        vue.number_input("Maximum hue delta", ns + "hue", default=0.05, required=False, visible=False),
        vue.number_input("Maximum white noise", ns + "stddev", default=0.01, required=False, visible=False),
        vue.number_input("Chance of augmentation steps", ns + "chance", default=0.5, required=False, visible=False)
    ]

# DEPRECATED
class ImageAugmentationSchema(vue.SchemaBuilderFunc):
    label = "Image Augmentation"
    ns = "augmentation"
    advanced = False
    module = ImageAugmenter

    def schema(self, b=vue.SchemaBuilder(), ns=ns, **kwargs):
        b += get_transform_schema(ns=ns+"transform.")
        b += get_noise_schema(ns=ns+"noise.")
        return b

Functions

def affine(o1, o2, r1, r2, sc, a, sh1, sh2, t1, t2)

Create N affine transform matrices from output to input (Nx3x3) for use with tfa.transform :param o1: Origin 1 (shape / 2 for center) :param o2: Origin 2 :param r1: Reference scale 1 (1 or -1 for flip) :param r2: Reference scale 1 :param sc: Augmented scale :param a: Rotation in radians :param sh1: Shear 1 :param sh2: Shear 2 :param t1: Translate 1 :param t2: Translate 2 :return: Inverse transformation matrix

Expand source code
@tf.function
def affine(o1, o2, r1, r2, sc, a, sh1, sh2, t1, t2):
    """
    Create N affine transform matrices from output to input (Nx3x3)
    for use with tfa.transform
    :param o1: Origin 1 (shape / 2 for center)
    :param o2: Origin 2
    :param r1: Reference scale 1 (1 or -1 for flip)
    :param r2: Reference scale 1
    :param sc: Augmented scale
    :param a: Rotation in radians
    :param sh1: Shear 1
    :param sh2: Shear 2
    :param t1: Translate 1
    :param t2: Translate 2
    :return: Inverse transformation matrix
    """
    _0 = tf.zeros_like(o1)
    _1 = tf.ones_like(o1)
    T1 = tf.transpose(tf.convert_to_tensor([[_1, _0, -(o2 - 1)],
                                            [_0, _1, -(o1 - 1)],
                                            [_0, _0, _1]], dtype=tf.float32), [2, 0, 1])
    T2 = tf.transpose(tf.convert_to_tensor([[_1, _0, (o2 - 1)],
                                            [_0, _1, (o1 - 1)],
                                            [_0, _0, _1]], dtype=tf.float32), [2, 0, 1])
    ref = tf.transpose(tf.convert_to_tensor([[r2, _0, _0],
                                             [_0, r1, _0],
                                             [_0, _0, _1]], dtype=tf.float32), [2, 0, 1])
    scale = tf.transpose(tf.convert_to_tensor([[1 + sc, _0, _0],
                                               [_0, 1 + sc, _0],
                                               [_0, _0, _1]], dtype=tf.float32), [2, 0, 1])
    rot = tf.transpose(tf.convert_to_tensor([[tf.cos(a), -tf.sin(a), _0],
                                             [tf.sin(a), tf.cos(a), _0],
                                             [_0, _0, _1]], dtype=tf.float32), [2, 0, 1])
    she = tf.transpose(tf.convert_to_tensor([[_1, sh2, _0],
                                             [sh1, _1, _0],
                                             [_0, _0, _1]], dtype=tf.float32), [2, 0, 1])
    tra = tf.transpose(tf.convert_to_tensor([[_1, _0, t2],
                                             [_0, _1, t1],
                                             [_0, _0, _1]], dtype=tf.float32), [2, 0, 1])
    return T2 @ tra @ she @ ref @ scale @ rot @ T1
def gaussian_blur(x, sigma)
Expand source code
@tf.function
def gaussian_blur(x, sigma):
    kernel_size = tf.cast(tf.math.ceil(sigma * 2.5), tf.int32) * 2 + 1
    gaussian_blur_kernel = tf.tile(_gaussian_kernel(kernel_size, sigma, x.dtype)[..., None, None],
                                   (1, 1, tf.shape(x)[-1], 1))
    x = tf.nn.depthwise_conv2d(x, gaussian_blur_kernel, [1, 1, 1, 1], 'SAME')
    return x
def get_noise_schema(ns)
Expand source code
def get_noise_schema(ns):
    return [
        vue.number_input("Maximum brightness delta", ns + "brightness", default=0.25, required=False, visible=False),
        vue.text_input("Emboss alpha range", ns + "emboss_alpha",
                       required=False, default="[0.0, 0.1]", json=True),
        vue.text_input("Emboss strength range", ns + "emboss_alpha",
                       required=False, default="[0.75, 1.25]", json=True),
        vue.text_input("Image saturation range", ns + "saturation",
                       required=False, default="[0, 2.0]", json=True),
        vue.number_input("Maximum hue delta", ns + "hue", default=0.05, required=False, visible=False),
        vue.number_input("Maximum white noise", ns + "stddev", default=0.01, required=False, visible=False),
        vue.number_input("Chance of augmentation steps", ns + "chance", default=0.5, required=False, visible=False)
    ]
def get_transform_schema(ns)
Expand source code
def get_transform_schema(ns):
    return [
        vue.number_input("Vertical translation range in percent", ns + "translate_vertical", default=0.1, required=False, visible=False),
        vue.number_input("Horizontal translation range in percent", ns + "translate_horizontal", default=0.1, required=False, visible=False),
        vue.number_input("Maximum scale offset", ns + "scale", default=0.2, required=False, visible=False),
        vue.number_input("Maximum relative shear", ns + "shear", default=0.04, required=False, visible=False),
        vue.number_input("Maximum rotation angle", ns + "rotate", default=90, required=False, visible=False),
        vue.checkbox("Flip up-down", ns + "flip_up_down", default=True, required=False, visible=False),
        vue.checkbox("Flip left-right", ns + "flip_left_right", default=True, required=False, visible=False),
        vue.number_input("Chance of augmentation steps", ns + "chance", default=0.5, required=False, visible=False)
    ]

Classes

class ImageAugmentationSchema (label='__DEFAULT__', ns='__DEFAULT__', advanced='__DEFAULT__')
Expand source code
class ImageAugmentationSchema(vue.SchemaBuilderFunc):
    label = "Image Augmentation"
    ns = "augmentation"
    advanced = False
    module = ImageAugmenter

    def schema(self, b=vue.SchemaBuilder(), ns=ns, **kwargs):
        b += get_transform_schema(ns=ns+"transform.")
        b += get_noise_schema(ns=ns+"noise.")
        return b

Ancestors

Class variables

var advanced
var label
var module

Base class for serializable modules

var ns

Inherited members

class ImageAugmenter (image_keys=None, label_keys=None, random_transformer: RandomTransformer = <brevettiai.data.image.image_augmenter.RandomTransformer object>, image_noise: ImageNoise = <brevettiai.data.image.image_augmenter.ImageNoise object>, image_filter: ImageFiltering = <brevettiai.data.image.image_augmenter.ImageFiltering object>, image_deformation: ImageDeformation = None)

Base class for serializable modules

Image augmentation class, for use with tensorflow and criterion datasets The call method expects a tensor dict with image and label keys for transformation. Alternatively, the augmenter may be used by calling transform_images directly

:param image_keys: labels of images to perform augmentation on :param label_keys: labels of annotations to perform augmentation on :param random_transformer: A random affine transformation object with RandomTransformer interfaces :param image_noise: An image noise generation object with ImageNoise interfaces :param image_filter: An image filter noise generation object with ImageFiltering interfaces :param image_deformation: A local random image deformation object with ImageDeformation interfaces

Expand source code
class ImageAugmenter(vue.VueSettingsModule):
    def __init__(self, image_keys=None, label_keys=None,
                 random_transformer: RandomTransformer = RandomTransformer(),
                 image_noise: ImageNoise = ImageNoise(),
                 image_filter: ImageFiltering = ImageFiltering(),
                 image_deformation: ImageDeformation = None):
        """
        Image augmentation class, for use with tensorflow and criterion datasets
        The call method expects a tensor dict with image and label keys for transformation.
        Alternatively, the augmenter may be used by calling transform_images directly

        :param image_keys: labels of images to perform augmentation on
        :param label_keys: labels of annotations to perform augmentation on
        :param random_transformer: A random affine transformation object with RandomTransformer interfaces
        :param image_noise: An image noise generation object with ImageNoise interfaces
        :param image_filter: An image filter noise generation object with ImageFiltering interfaces
        :param image_deformation: A local random image deformation object with ImageDeformation interfaces

        """
        self.image_keys = image_keys or ["img"]
        self.label_keys = label_keys or ["segmentation", "annotation"]
        self.random_transformer = random_transformer
        self.image_noise = image_noise
        self.image_filter = image_filter
        self.image_deformation = image_deformation
        pass

    def __call__(self, x, seed, *args, **kwargs):
        A = deformations = probabilities = None

        if self.random_transformer is not None:
            A = self.random_transformer(tf.shape(x[self.image_keys[0]]), seed=seed)
        if self.image_deformation is not None:
            deformations, probabilities = self.image_deformation(tf.shape(x[self.image_keys[0]]), seed)

        for key in self.image_keys:
            if key in x:
                if self.random_transformer is not None:
                    x[key] = self.random_transformer.transform_images(x[key], A)
                if self.image_deformation is not None:
                    x[key] = self.image_deformation.apply(x[key], deformations, probabilities)
                if self.image_filter is not None:
                    x[key] = self.image_filter(x[key], seed=seed)
                if self.image_noise is not None:
                    x[key] = self.image_noise(x[key], seed=seed)

        for key in self.label_keys:
            if key in x:
                if self.random_transformer is not None:
                    x[key] = self.random_transformer.transform_images(tf.cast(x[key], tf.float32), A,
                                                                      interpolation="nearest", fill_seg=True)
                if self.image_deformation is not None:
                    x[key] = self.image_deformation.apply(tf.cast(x[key], tf.float32), deformations, probabilities, interpolation="nearest")
        return x

    @classmethod
    def to_schema(cls, builder, name, ptype, default, **kwargs):

        if name in {"image_keys", "label_keys"}:
            return
        else:
            return super().to_schema(builder=builder, name=name, ptype=ptype, default=default, **kwargs)

Ancestors

Inherited members

class ImageDeformation (alpha: float = 0.0, sigma: float = 0.5, chance: float = 0.5)

Base class for serializable modules

Expand source code
class ImageDeformation(vue.VueSettingsModule):
    def __init__(self, alpha: float = 0.0, sigma: float = 0.5, chance: float = 0.5):
        self.alpha = alpha
        self.sigma = sigma
        self.chance = chance

    def __call__(self, shape, seed):
        deformations = tf.random.uniform((shape[0], shape[1], shape[2], 2), seed=seed+114)
        probabilities = tf.random.uniform((shape[0], ), seed=seed + 113)
        return deformations, probabilities

    def deform_image(self, inputs, interpolation="bilinear"):
        x, dxy, do_deform = inputs
        if do_deform < self.chance:
            shape = tf.shape(x)
            dxy = gaussian_blur(dxy[None] * 2 - 1, self.sigma) * self.alpha
            if interpolation and interpolation == "nearest":
                dxy = tf.round(dxy)

            gx, gy = tf.meshgrid(tf.range(shape[1], dtype=tf.float32), tf.range(shape[0], dtype=tf.float32), indexing="xy")
            grid = tf.stack([gx + dxy[..., 0],
                             gy + dxy[..., 1]], -1)
            return tfa.image.resampler(x[None], grid)[0]
        else:
            return x

    def apply(self, x, deformations, probabilities, interpolation="bilinear"):
        """Elastic deformation of images as described in [Simard2003]_ (with modifications).
        .. [Simard2003] Simard, Steinkraus and Platt, "Best Practices for
             Convolutional Neural Networks applied to Visual Document Analysis", in
             Proc. of the International Conference on Document Analysis and
             Recognition, 2003.

         Based on https://gist.github.com/erniejunior/601cdf56d2b424757de5
        """

        deformation = tf.map_fn(fn=partial(self.deform_image, interpolation=interpolation),
                                elems=(x, deformations,probabilities), dtype=x.dtype)
        return deformation

Ancestors

Methods

def apply(self, x, deformations, probabilities, interpolation='bilinear')

Elastic deformation of images as described in [Simard2003]_ (with modifications). .. [Simard2003] Simard, Steinkraus and Platt, "Best Practices for Convolutional Neural Networks applied to Visual Document Analysis", in Proc. of the International Conference on Document Analysis and Recognition, 2003.

Based on https://gist.github.com/erniejunior/601cdf56d2b424757de5

Expand source code
def apply(self, x, deformations, probabilities, interpolation="bilinear"):
    """Elastic deformation of images as described in [Simard2003]_ (with modifications).
    .. [Simard2003] Simard, Steinkraus and Platt, "Best Practices for
         Convolutional Neural Networks applied to Visual Document Analysis", in
         Proc. of the International Conference on Document Analysis and
         Recognition, 2003.

     Based on https://gist.github.com/erniejunior/601cdf56d2b424757de5
    """

    deformation = tf.map_fn(fn=partial(self.deform_image, interpolation=interpolation),
                            elems=(x, deformations,probabilities), dtype=x.dtype)
    return deformation
def deform_image(self, inputs, interpolation='bilinear')
Expand source code
def deform_image(self, inputs, interpolation="bilinear"):
    x, dxy, do_deform = inputs
    if do_deform < self.chance:
        shape = tf.shape(x)
        dxy = gaussian_blur(dxy[None] * 2 - 1, self.sigma) * self.alpha
        if interpolation and interpolation == "nearest":
            dxy = tf.round(dxy)

        gx, gy = tf.meshgrid(tf.range(shape[1], dtype=tf.float32), tf.range(shape[0], dtype=tf.float32), indexing="xy")
        grid = tf.stack([gx + dxy[..., 0],
                         gy + dxy[..., 1]], -1)
        return tfa.image.resampler(x[None], grid)[0]
    else:
        return x

Inherited members

class ImageFiltering (emboss_strength: tuple = None, avg_blur: tuple = (3, 3), gaussian_blur_sigma: float = 0.5, chance: float = 0.5)

Base class for serializable modules

:param emboss_strength: Embossing in a random direction with a scale chosen in the given range :param avg_blur: Size of mean filtering kernel :param gaussian_blur_sigma: :param chance: The chance of the individual step to be applied

Expand source code
class ImageFiltering(vue.VueSettingsModule):
    def __init__(self, emboss_strength: tuple = None, avg_blur: tuple = (3, 3), gaussian_blur_sigma: float = 0.5,
                 chance: float = 0.5):
        """
        :param emboss_strength: Embossing in a random direction with a scale chosen in the given range
        :param avg_blur: Size of mean filtering kernel
        :param gaussian_blur_sigma:
        :param chance: The chance of the individual step to be applied
        """
        self.emboss_strength = emboss_strength or (0, 0.25)
        self.avg_blur = avg_blur or (3, 3)
        self.gaussian_blur_sigma = gaussian_blur_sigma
        self.chance = chance

    def conditional_filter(self, inputs):
        x, probability, emboss_xy = inputs
        if probability < self.chance / 3:
            # Emboss
            dxy = tf.image.sobel_edges(x[None])[0]
            emboss = tf.reduce_sum(emboss_xy * dxy, axis=-1)
            return x + emboss
        elif probability < 2 * self.chance / 3:
            # Gaussian blur
            return gaussian_blur(x[None], self.gaussian_blur_sigma)[0]
        elif probability < 3 * self.chance / 3:
            # Average blur
            avg_blur_kernel =  tf.tile(_avg_kernel(self.avg_blur, x.dtype)[..., None, None], (1, 1, tf.shape(x)[-1], 1))
            return tf.nn.depthwise_conv2d(x[None], avg_blur_kernel, [1, 1, 1, 1], 'SAME')[0]
        else:
            # No filtering
            return x

    def __call__(self, x, seed):
        sh = tf.shape(x)
        probabilities = tf.random.uniform((sh[0],), seed=seed+102)

        emboss_sign = tf.sign(tf.random.uniform((sh[0], 1, 1, sh[3], 2), -1, 1, seed=seed + 103))
        emboss_xy = emboss_sign * tf.random.uniform((sh[0], 1, 1, sh[3], 2), self.emboss_strength[0],
                                                    self.emboss_strength[1], seed=seed + 104)
        return tf.map_fn(self.conditional_filter, [x, probabilities, emboss_xy], dtype=tf.float32)

Ancestors

Methods

def conditional_filter(self, inputs)
Expand source code
def conditional_filter(self, inputs):
    x, probability, emboss_xy = inputs
    if probability < self.chance / 3:
        # Emboss
        dxy = tf.image.sobel_edges(x[None])[0]
        emboss = tf.reduce_sum(emboss_xy * dxy, axis=-1)
        return x + emboss
    elif probability < 2 * self.chance / 3:
        # Gaussian blur
        return gaussian_blur(x[None], self.gaussian_blur_sigma)[0]
    elif probability < 3 * self.chance / 3:
        # Average blur
        avg_blur_kernel =  tf.tile(_avg_kernel(self.avg_blur, x.dtype)[..., None, None], (1, 1, tf.shape(x)[-1], 1))
        return tf.nn.depthwise_conv2d(x[None], avg_blur_kernel, [1, 1, 1, 1], 'SAME')[0]
    else:
        # No filtering
        return x

Inherited members

class ImageNoise (brightness: float = 0.25, contrast: tuple = (0.5, 1.25), hue: float = 0.05, saturation: tuple = (0, 2.0), stddev: float = 0.01, chance: float = 0.5)

Base class for serializable modules

:param brightness: Embossing in a random direction with a scale chosen in the given range :param contrast: Size of mean filtering kernel :param hue: :param saturation: :param stddev: :param chance: The chance of the individual step to be applied

Expand source code
class ImageNoise(vue.VueSettingsModule):
    def __init__(self, brightness: float = 0.25, contrast: tuple = (0.5, 1.25),
                 hue: float = 0.05, saturation: tuple = (0, 2.0), stddev: float = 0.01, chance: float = 0.5):
        """
        :param brightness: Embossing in a random direction with a scale chosen in the given range
        :param contrast: Size of mean filtering kernel
        :param hue:
        :param saturation:
        :param stddev:
        :param chance: The chance of the individual step to be applied
        """
        self.brightness = brightness
        self.contrast = contrast or (0.5, 1.25)
        self.hue = hue
        self.saturation = saturation or (0, 2.0)
        self.stddev = stddev
        self.chance = chance

    def conditional_noise(self, inputs):
        x, hue, saturation, prob_hue_sat, contrast, prob_contrast, brightness, prob_brightness, noise, prob_noise = inputs
        sh = tf.shape(x)

        if sh[2] == 3 and prob_hue_sat < self.chance:
            x = tf.image.adjust_saturation(x, saturation)
            x = tf.image.adjust_hue(x, hue)

        if prob_contrast < self.chance:
            x = tf.image.adjust_contrast(x, contrast)

        if prob_brightness < self.chance:
            x = tf.image.adjust_brightness(x, brightness)

        if prob_noise < self.chance:
            x = tf.add(x, noise)
        return x

    def __call__(self, x, seed):
        sh = tf.shape(x)

        hue = tf.random.uniform((sh[0], ), -self.hue, self.hue, seed=seed + 100)
        saturation = tf.random.uniform((sh[0], ), self.saturation[0], self.saturation[1], seed=seed+101)
        prob_hue_sat = tf.random.uniform((sh[0],), seed=seed+102)

        contrast = tf.random.uniform((sh[0], ), self.contrast[0], self.contrast[1], seed=seed+103)
        prob_contrast = tf.random.uniform((sh[0],), seed=seed+104)

        brightness = tf.random.uniform((sh[0], ), -self.brightness, self.brightness, seed=seed+105)
        prob_brightness = tf.random.uniform((sh[0],), seed=seed+106)

        noise = tf.random.normal(shape=tf.shape(x), mean=0, stddev=self.stddev,
                                 dtype=tf.float32, seed=seed+107)
        prob_noise = tf.random.uniform((sh[0],), seed=seed+108)

        return tf.map_fn(self.conditional_noise, [x,
                                                  hue, saturation, prob_hue_sat,
                                                  contrast, prob_contrast,
                                                  brightness, prob_brightness,
                                                  noise, prob_noise], dtype=tf.float32)

Ancestors

Methods

def conditional_noise(self, inputs)
Expand source code
def conditional_noise(self, inputs):
    x, hue, saturation, prob_hue_sat, contrast, prob_contrast, brightness, prob_brightness, noise, prob_noise = inputs
    sh = tf.shape(x)

    if sh[2] == 3 and prob_hue_sat < self.chance:
        x = tf.image.adjust_saturation(x, saturation)
        x = tf.image.adjust_hue(x, hue)

    if prob_contrast < self.chance:
        x = tf.image.adjust_contrast(x, contrast)

    if prob_brightness < self.chance:
        x = tf.image.adjust_brightness(x, brightness)

    if prob_noise < self.chance:
        x = tf.add(x, noise)
    return x

Inherited members

class ImageSaltAndPepper (fraction: float = 0.0002, value_range: tuple = None, scale: int = 1, chance: float = 0.5)

Base class for serializable modules

:param brightness: Embossing in a random direction with a scale chosen in the given range :param contrast: Size of mean filtering kernel :param hue: :param saturation: :param stddev: :param chance: The chance of the individual step to be applied

Expand source code
class ImageSaltAndPepper(vue.VueSettingsModule):
    def __init__(self, fraction: float = 0.0002, value_range: tuple = None, scale: int = 1, chance: float = 0.5):
        """
        :param brightness: Embossing in a random direction with a scale chosen in the given range
        :param contrast: Size of mean filtering kernel
        :param hue:
        :param saturation:
        :param stddev:
        :param chance: The chance of the individual step to be applied
        """
        self.fraction = fraction
        self.value_range = value_range or (0.0, 1.0)
        self.scale = scale
        assert scale in [1, 3, 5]
        self.chance = chance

        self.fractions = (self.fraction / 2, 1.0-self.fraction / 2)

    def apply_noise(self, inputs):
        x, salt_pepper, prob_salt_pepper = inputs
        if prob_salt_pepper < self.chance:
            if self.scale == 5:
                sp = tf.cast(-tf.nn.max_pool2d(-salt_pepper[None], (1, 5, 5, 1), (1, 1, 1, 1), "SAME") < self.fractions[0], tf.float32)[0] * \
                     self.value_range[0] + \
                     tf.cast(tf.nn.max_pool2d(salt_pepper[None], (1, 5, 5, 1), (1, 1 ,1, 1), "SAME") > self.fractions[1], tf.float32)[0] * \
                     self.value_range[1]
            elif self.scale == 3:
                sp = tf.cast(-tf.nn.max_pool2d(-salt_pepper[None], (1, 3, 3, 1), (1, 1, 1, 1), "SAME") < self.fractions[0], tf.float32)[0] * \
                     self.value_range[0] + \
                     tf.cast(tf.nn.max_pool2d(salt_pepper[None], (1, 3, 3, 1), (1, 1 ,1, 1), "SAME") > self.fractions[1], tf.float32)[0] * \
                     self.value_range[1]
            elif self.scale == 1:
                sp = tf.cast(salt_pepper < self.fractions[0], tf.float32) * self.value_range[0] + \
                     tf.cast(salt_pepper > self.fractions[1], tf.float32) * self.value_range[1]
            return sp
        else:
            return tf.zeros_like(salt_pepper)

    def __call__(self, x, seed):
        sh = tf.shape(x)

        salt_pepper = tf.random.uniform(shape=sh[:3], minval=0.0, maxval=1.0, dtype=tf.float32, seed=seed+111)[..., None]

        prob_salt_pepper = tf.random.uniform((sh[0], ), seed=seed+112)
        return tf.map_fn(self.apply_noise, [x, salt_pepper, prob_salt_pepper], dtype=tf.float32)

Ancestors

Methods

def apply_noise(self, inputs)
Expand source code
def apply_noise(self, inputs):
    x, salt_pepper, prob_salt_pepper = inputs
    if prob_salt_pepper < self.chance:
        if self.scale == 5:
            sp = tf.cast(-tf.nn.max_pool2d(-salt_pepper[None], (1, 5, 5, 1), (1, 1, 1, 1), "SAME") < self.fractions[0], tf.float32)[0] * \
                 self.value_range[0] + \
                 tf.cast(tf.nn.max_pool2d(salt_pepper[None], (1, 5, 5, 1), (1, 1 ,1, 1), "SAME") > self.fractions[1], tf.float32)[0] * \
                 self.value_range[1]
        elif self.scale == 3:
            sp = tf.cast(-tf.nn.max_pool2d(-salt_pepper[None], (1, 3, 3, 1), (1, 1, 1, 1), "SAME") < self.fractions[0], tf.float32)[0] * \
                 self.value_range[0] + \
                 tf.cast(tf.nn.max_pool2d(salt_pepper[None], (1, 3, 3, 1), (1, 1 ,1, 1), "SAME") > self.fractions[1], tf.float32)[0] * \
                 self.value_range[1]
        elif self.scale == 1:
            sp = tf.cast(salt_pepper < self.fractions[0], tf.float32) * self.value_range[0] + \
                 tf.cast(salt_pepper > self.fractions[1], tf.float32) * self.value_range[1]
        return sp
    else:
        return tf.zeros_like(salt_pepper)

Inherited members

class RandomTransformer (chance: float = 0.5, flip_up_down: bool = True, flip_left_right: bool = True, scale: float = 0.2, rotate_chance: float = 0.5, rotate: float = 90, translate_horizontal: float = 0.1, translate_vertical: float = 0.1, shear: float = 0.04, interpolation: str = 'bilinear')

Base class for serializable modules

Build random transformation matrices for batch of images :param shape: :param chance: :param flip: :param scale: :param rotate: :param translate: :param shear: :param interpolation: Resampling interpolation method :return:

Expand source code
class RandomTransformer(vue.VueSettingsModule):
    def __init__(self, chance: float = 0.5, flip_up_down: bool = True, flip_left_right: bool = True,
                 scale: float = 0.2, rotate_chance: float = 0.5, rotate: float = 90,
                 translate_horizontal: float = 0.1, translate_vertical: float = 0.1, shear: float = 0.04,
                 interpolation: str = "bilinear"):
        """
        Build random transformation matrices for batch of images
        :param shape:
        :param chance:
        :param flip:
        :param scale:
        :param rotate:
        :param translate:
        :param shear:
        :param interpolation: Resampling interpolation method
        :return:
        """

        self.chance = chance
        self.flip_up_down = flip_up_down
        self.flip_left_right = flip_left_right
        self.scale = scale
        self.rotate_chance = rotate_chance
        self.rotate = rotate
        self.translate_horizontal = translate_horizontal
        self.translate_vertical = translate_vertical
        self.shear = shear
        self.interpolation = interpolation
        self.fill_seg_value = 0.0

    def set_fill_seg_value(self, fill_value):
        self.fill_seg_value = fill_value

    def transform_images(self, x, A, interpolation=None, fill_seg=False):
        """
        :param x: 4D image tensor (batch_size x height x width x channels)
        :param A: 3D stack of affine (batch_size x 3 x 3) type is always float32
        """

        tr = tfa.image.transform_ops.matrices_to_flat_transforms(A)
        x = tfa.image.transform(x, tr, interpolation or self.interpolation)

        if fill_seg:
            mask = tfa.image.transform(tf.zeros_like(x[..., :1]), tr, interpolation or self.interpolation,
                                       fill_value=0.0)
            x += mask * self.fill_seg_value

        return x

    def __call__(self, shape, seed):
        sh = shape
        origin = tf.cast(sh[1:3], np.float32)[None] / 2 * tf.ones((sh[0], 1))

        flip = (self.flip_up_down, self.flip_left_right)

        mask_rotate = tf.cast(tf.random.uniform(sh[:1], seed=seed) < self.rotate_chance, tf.float32)

        rotate = self.rotate
        try:
            iter(rotate)
        except TypeError:
            rotate = [-rotate, rotate]
        rotate = np.array(rotate) * np.pi / 180
        rotate = tf.random.uniform(sh[:1], *rotate, seed=seed + 1) * mask_rotate

        mask_flip = tf.cast(tf.random.uniform(sh[:1], seed=seed) < self.chance, tf.float32)
        if flip[0]:
            flip1 = tf.sign(tf.random.uniform(sh[:1], -1, 1, seed=seed + 2)) * mask_flip + (1-mask_flip)
        else:
            flip1 = tf.ones_like(origin[:, 0])
        if flip[1]:
            flip2 = tf.sign(tf.random.uniform(sh[:1], -1, 1, seed=seed + 3)) * mask_flip + (1-mask_flip)
        else:
            flip2 = tf.ones_like(origin[:, 0])

        mask_scale_shear = tf.cast(tf.random.uniform(sh[:1], seed=seed + 4) < self.chance, tf.float32)
        scale = self.scale
        try:
            iter(scale)
        except TypeError:
            scale = [-scale, scale]
        scale = tf.random.uniform(sh[:1], *scale, seed=seed + 5) * mask_scale_shear
        shear = self.shear
        try:
            iter(shear)
        except TypeError:
            shear = [-shear, shear]
        shear1 = tf.random.uniform(sh[:1], *shear, seed=seed + 8) * mask_scale_shear
        shear2 = tf.random.uniform(sh[:1], *shear, seed=seed + 9) * mask_scale_shear

        mask_translate = tf.cast(tf.random.uniform(sh[:1], seed=seed) < self.chance, tf.float32)
        translate = [[-self.translate_vertical, self.translate_vertical],
                     [-self.translate_horizontal, self.translate_horizontal]]

        translate = tf.convert_to_tensor(translate) * tf.cast(sh[1:3], tf.float32)[:, None]
        translate1 = tf.random.uniform(sh[:1], translate[0, 0], translate[0, 1], seed=seed + 6) * mask_translate
        translate2 = tf.random.uniform(sh[:1], translate[1, 0], translate[1, 1], seed=seed + 7) * mask_translate

        A = affine(origin[:, 0], origin[:, 1], flip1, flip2, scale, rotate, shear1, shear2, translate1, translate2)
        return A

Ancestors

Methods

def set_fill_seg_value(self, fill_value)
Expand source code
def set_fill_seg_value(self, fill_value):
    self.fill_seg_value = fill_value
def transform_images(self, x, A, interpolation=None, fill_seg=False)

:param x: 4D image tensor (batch_size x height x width x channels) :param A: 3D stack of affine (batch_size x 3 x 3) type is always float32

Expand source code
def transform_images(self, x, A, interpolation=None, fill_seg=False):
    """
    :param x: 4D image tensor (batch_size x height x width x channels)
    :param A: 3D stack of affine (batch_size x 3 x 3) type is always float32
    """

    tr = tfa.image.transform_ops.matrices_to_flat_transforms(A)
    x = tfa.image.transform(x, tr, interpolation or self.interpolation)

    if fill_seg:
        mask = tfa.image.transform(tf.zeros_like(x[..., :1]), tr, interpolation or self.interpolation,
                                   fill_value=0.0)
        x += mask * self.fill_seg_value

    return x

Inherited members

class ViewGlimpseFromBBox (bbox_key=None, target_shape: tuple = None, zoom_factor: int = None)

Base class for serializable modules

Expand source code
class ViewGlimpseFromBBox(vue.VueSettingsModule):
    def __init__(self, bbox_key=None,
                 target_shape: tuple = None, zoom_factor: int = None):
        self.bbox_key = bbox_key or ImageKeys.BOUNDING_BOX
        self.target_shape = target_shape

        # Ensure zoom_factor is not 0
        self.zoom_factor = None if zoom_factor == 0 else zoom_factor

        if self.zoom_factor:
            assert np.log2(self.zoom_factor).is_integer(), \
                f"zoom_exp_factor '{self.zoom_factor}' must be a power of 2"

    @property
    def bbox_shape(self):
        if self.zoom_factor: # not zero or None
            return tuple(int(x * self.zoom_factor) for x in self.target_shape)
        else:
            return self.target_shape

    def __call__(self, x, seed):
        # Extract info
        img_size = tf.cast(x[ImageKeys.SIZE], tf.int32)
        views = tf.cast(x[self.bbox_key], tf.int32)

        # Choose glimpse in bbox
        offset_scales = views[:, 2:] - views[:, :2] + self.bbox_shape
        offsets = tf.random.uniform(tf.shape(offset_scales), seed=seed + 201) * tf.cast(offset_scales, tf.float32)
        view_center = views[:, :2] + tf.cast(tf.round(offsets), tf.int32)
        view_origin = view_center - self.bbox_shape

        # Clip bbox to boundaries of image
        max_idx = img_size - self.bbox_shape
        min_idx = tf.zeros_like(max_idx, dtype=tf.int32)
        view_origin = tf.clip_by_value(view_origin, min_idx, max_idx)

        # Update bbox
        views = tf.concat([view_origin, view_origin + tf.constant(self.bbox_shape)[None]], axis=1)
        x[self.bbox_key] = views

        # Add zoom to dataset
        if self.zoom_factor is not None:
            # Create tensor of shape (batch_size,) with value self.zoom_factor
            zef = tf.constant(self.zoom_factor, dtype=tf.float32, shape=(1,))
            x[ImageKeys.ZOOM] = tf.broadcast_to(zef, tf.shape(x[self.bbox_key])[:1])
        return x

Ancestors

Instance variables

var bbox_shape
Expand source code
@property
def bbox_shape(self):
    if self.zoom_factor: # not zero or None
        return tuple(int(x * self.zoom_factor) for x in self.target_shape)
    else:
        return self.target_shape

Inherited members

class ViewGlimpseFromPoints (bbox_key=None, target_shape: tuple = None, zoom_factor: int = None, overlap: int = 0.8)

Base class for serializable modules

Expand source code
class ViewGlimpseFromPoints(vue.VueSettingsModule):
    def __init__(self, bbox_key=None, target_shape: tuple = None, zoom_factor: int = None, overlap: int = 0.8):
        self.bbox_key = bbox_key or ImageKeys.BOUNDING_BOX
        self.target_shape = target_shape

        # Ensure zoom_factor is not 0
        self.zoom_factor = None if zoom_factor == 0 else zoom_factor

        self.overlap = overlap

        if self.zoom_factor and not np.log2(self.zoom_factor).is_integer():
            log.warning(f"zoom_exp_factor '{self.zoom_factor}' must be a power of 2 for tiled images to work")

    @property
    def bbox_shape(self):
        if self.zoom_factor: # not zero or None
            return tuple(int(x * self.zoom_factor) for x in self.target_shape)
        else:
            return self.target_shape

    def __call__(self, x, seed):
        inside_points = x[ImageKeys.INSIDE_POINTS]
        sh = tf.shape(inside_points)

        if ImageKeys.BBOX_SIZE_ADJUST in x:
            bbox_adjust = tf.cast(x[ImageKeys.BBOX_SIZE_ADJUST], tf.float32)
            adjusted_bbox_shape = bbox_adjust[:, None] * self.bbox_shape
            x[ImageKeys.ZOOM] = tf.round(adjusted_bbox_shape)[:, 0] / self.bbox_shape[0]
        else:
            adjusted_bbox_shape = tf.constant(self.bbox_shape, dtype=tf.float32)[None]

        # Choose target point
        choice = tf.random.uniform(shape=(sh[0],), maxval=sh[1], dtype=tf.int32, seed=seed + 202)
        target_points = tf.gather_nd(inside_points, indices=tf.stack([tf.range(sh[0]), choice], axis=-1))

        # Adjust bbox
        half_size = tf.cast(tf.round(adjusted_bbox_shape / 2), tf.int32)
        overlap_px = tf.cast(half_size, tf.float32) * self.overlap
        offset = tf.random.uniform(tf.shape(target_points), minval=-1, maxval=1, seed=seed + 201) * overlap_px
        view_origin = target_points - half_size + tf.cast(offset, tf.int32)

        # Clip bbox to boundaries of image
        img_size = tf.cast(x[ImageKeys.SIZE], tf.int32)
        max_idx = img_size - self.bbox_shape
        min_idx = tf.zeros_like(max_idx, dtype=tf.int32)
        view_origin = tf.clip_by_value(view_origin, min_idx, max_idx)

        # Update bbox
        views = tf.concat([view_origin, view_origin + tf.cast(tf.round(adjusted_bbox_shape), tf.int32)], axis=1)
        x[self.bbox_key] = views

        # Add zoom to dataset
        if self.zoom_factor is not None:
            # Create tensor of shape (batch_size,) with value self.zoom_factor
            zef = tf.constant(self.zoom_factor, dtype=tf.float32, shape=(1,))
            zef = tf.broadcast_to(zef, tf.shape(x[self.bbox_key])[:1])
            x[ImageKeys.ZOOM] = zef * x[ImageKeys.ZOOM] if ImageKeys.ZOOM in x else zef

        return x

Ancestors

Instance variables

var bbox_shape
Expand source code
@property
def bbox_shape(self):
    if self.zoom_factor: # not zero or None
        return tuple(int(x * self.zoom_factor) for x in self.target_shape)
    else:
        return self.target_shape

Inherited members