Adding noise perturbation detector with Gaussian noise (#52)

* ENH: adding noise perturbation detector with Gaussian noise

* FIX: docstring

* FIX: the IoU metric now handles prediction batches

* renaming metric IoU to IoUMean

---------

Co-authored-by: Rabah Khalek <rabah.khalek@gmail.com>

giskard_vision/core/dataloaders/wrappers.py

@@ -228,6 +228,78 @@ def get_image(self, idx: int) -> np.ndarray:
         return cv2.GaussianBlur(image, self._kernel_size, *self._sigma)
 
 
+class NoisyDataLoader(DataLoaderWrapper):
+    """Wrapper class for a DataIteratorBase, providing noisy images.
+
+    Args:
+        dataloader (DataIteratorBase): The data loader to be wrapped.
+        sigma (float): Standard deviation of the Gaussian noise.
+
+    Returns:
+        NoisyDataLoader: Noisy data loader instance.
+    """
+
+    def __init__(
+        self,
+        dataloader: DataIteratorBase,
+        sigma: float = 0.1,
+    ) -> None:
+        """
+        Initializes the BlurredDataLoader.
+
+        Args:
+            dataloader (DataIteratorBase): The data loader to be wrapped.
+            sigma (float): Standard deviation of the Gaussian noise.
+        """
+        super().__init__(dataloader)
+        self._sigma = sigma
+
+    @property
+    def name(self):
+        """
+        Gets the name of the blurred data loader.
+
+        Returns:
+            str: The name of the blurred data loader.
+        """
+        return "noisy"
+
+    def get_image(self, idx: int) -> np.ndarray:
+        """
+        Gets a blurred image using Gaussian blur.
+
+        Args:
+            idx (int): Index of the data.
+
+        Returns:
+            np.ndarray: Blurred image data.
+        """
+        image = super().get_image(idx)
+        return self.add_gaussian_noise(image, self._sigma * 255)
+
+    def add_gaussian_noise(self, image, std_dev):
+        """
+        Add Gaussian noise to the image
+
+        Args:
+            image (np.ndarray): Image
+            std_dev (float): Standard deviation of the Gaussian noise.
+
+        Returns:
+            np.ndarray: Noisy image
+        """
+        # Generate Gaussian noise
+        noise = np.random.normal(0, std_dev, image.shape).astype(np.float32)
+
+        # Add the noise to the image
+        noisy_image = cv2.add(image.astype(np.float32), noise)
+
+        # Clip the values to stay within valid range (0-255 for uint8)
+        noisy_image = np.clip(noisy_image, 0, 255).astype(np.uint8)
+
+        return noisy_image
+
+
 class ColoredDataLoader(DataLoaderWrapper):
     """Wrapper class for a DataIteratorBase, providing color-altered images using OpenCV color conversion.
 

giskard_vision/core/detectors/metrics.py

@@ -1,9 +1,9 @@
 from giskard_vision.image_classification.tests.performance import Accuracy
 from giskard_vision.landmark_detection.tests.performance import NMEMean
-from giskard_vision.object_detection.tests.performance import IoU
+from giskard_vision.object_detection.tests.performance import IoUMean
 
 detector_metrics = {
     "image_classification": Accuracy,
     "landmark": NMEMean,
-    "object_detection": IoU,
+    "object_detection": IoUMean,
 }

giskard_vision/core/detectors/transformation_noise_detector.py

@@ -0,0 +1,23 @@
+from giskard_vision.core.dataloaders.wrappers import NoisyDataLoader
+
+from ...core.detectors.decorator import maybe_detector
+from .perturbation import PerturbationBaseDetector, Robustness
+
+
+@maybe_detector("noise", tags=["vision", "robustness", "image_classification", "landmark", "object_detection"])
+class TransformationNoiseDetectorLandmark(PerturbationBaseDetector):
+    """
+    Detector that evaluates models performance on noisy images
+    """
+
+    issue_group = Robustness
+
+    def __init__(self, sigma=0.1):
+        self.sigma = sigma
+
+    def get_dataloaders(self, dataset):
+        dl = NoisyDataLoader(dataset, self.sigma)
+
+        dls = [dl]
+
+        return dls

giskard_vision/core/tests/base.py

@@ -314,7 +314,7 @@ def run(
             prediction_time=prediction_time,
             prediction_fail_rate=prediction_fail_rate,
             metric_name=self.metric.name,
-            model_name=model.name,
+            model_name=model.name if hasattr(model, "name") else model.__class__.__name__,
             dataloader_name=dataloader.name,
             dataloader_ref_name=dataloader_ref.name,
             indexes_examples=indexes,

giskard_vision/object_detection/detectors/metadata_detector.py

@@ -15,7 +15,7 @@
     SurrogateRelativeTopLeftY,
     SurrogateStdIntensity,
 )
-from giskard_vision.object_detection.tests.performance import IoU
+from giskard_vision.object_detection.tests.performance import IoUMean
 
 from ...core.detectors.decorator import maybe_detector
 
@@ -38,7 +38,7 @@ class MetaDataScanDetectorObjectDetection(MetaDataScanDetector):
         SurrogateRelativeTopLeftY,
         SurrogateNormalizedPerimeter,
     ]
-    metric = IoU
+    metric = IoUMean
     type_task = "regression"
     metric_type = "absolute"
     metric_direction = "better_higher"

giskard_vision/object_detection/tests/performance.py

@@ -1,48 +1,59 @@
 from dataclasses import dataclass
 
+import numpy as np
+
 from ..types import Types
 from .base import Metric
 
 
 @dataclass
-class IoU(Metric):
+class IoUMean(Metric):
     """Intersection over Union distance between a prediction and a ground truth"""
 
-    name = "IoU"
+    name = "IoUMean"
     description = "Intersection over Union"
 
     @staticmethod
     def definition(prediction_result: Types.prediction_result, ground_truth: Types.label):
-
         # if prediction_result.prediction.item().get("labels") != ground_truth.item().get("labels"):
         #     return 0
 
-        gt_box = prediction_result.prediction.item().get("boxes")
-        pred_box = ground_truth.item().get("boxes")
+        ious = []
+
+        for i in range(len(prediction_result.prediction)):
+            if isinstance(prediction_result.prediction[i], dict):
+                gt_box = prediction_result.prediction[i].get("boxes")
+            else:
+                ious.append(0)
+                continue
+
+            pred_box = ground_truth[i].get("boxes")
+
+            x1_min, y1_min, x1_max, y1_max = gt_box
+            x2_min, y2_min, x2_max, y2_max = pred_box
 
-        x1_min, y1_min, x1_max, y1_max = gt_box
-        x2_min, y2_min, x2_max, y2_max = pred_box
+            # Calculate the coordinates of the intersection rectangle
+            x_inter_min = max(x1_min, x2_min)
+            y_inter_min = max(y1_min, y2_min)
+            x_inter_max = min(x1_max, x2_max)
+            y_inter_max = min(y1_max, y2_max)
 
-        # Calculate the coordinates of the intersection rectangle
-        x_inter_min = max(x1_min, x2_min)
-        y_inter_min = max(y1_min, y2_min)
-        x_inter_max = min(x1_max, x2_max)
-        y_inter_max = min(y1_max, y2_max)
+            # Compute the area of the intersection rectangle
+            if x_inter_max < x_inter_min or y_inter_max < y_inter_min:
+                inter_area = 0
+            else:
+                inter_area = (x_inter_max - x_inter_min) * (y_inter_max - y_inter_min)
 
-        # Compute the area of the intersection rectangle
-        if x_inter_max < x_inter_min or y_inter_max < y_inter_min:
-            inter_area = 0
-        else:
-            inter_area = (x_inter_max - x_inter_min) * (y_inter_max - y_inter_min)
+            # Compute the area of both the prediction and ground-truth rectangles
+            box1_area = (x1_max - x1_min) * (y1_max - y1_min)
+            box2_area = (x2_max - x2_min) * (y2_max - y2_min)
 
-        # Compute the area of both the prediction and ground-truth rectangles
-        box1_area = (x1_max - x1_min) * (y1_max - y1_min)
-        box2_area = (x2_max - x2_min) * (y2_max - y2_min)
+            # Compute the union area
+            union_area = box1_area + box2_area - inter_area
 
-        # Compute the union area
-        union_area = box1_area + box2_area - inter_area
+            # Compute the IoU
+            iou = inter_area / union_area
 
-        # Compute the IoU
-        iou = inter_area / union_area
+            ious.append(iou)
 
-        return iou
+        return np.mean(ious)