From 1f4600f7395e8e4313777ee5707ebf5b93c2ea86 Mon Sep 17 00:00:00 2001 From: Ashwin Vaidya Date: Fri, 20 Sep 2024 15:27:15 +0200 Subject: [PATCH 1/3] remove numba Signed-off-by: Ashwin Vaidya --- pyproject.toml | 3 +- src/anomalib/metrics/per_image/__init__.py | 3 +- .../metrics/per_image/_binclf_curve_numba.py | 115 ------------------ src/anomalib/metrics/per_image/_validate.py | 4 +- .../metrics/per_image/binclf_curve.py | 5 +- .../metrics/per_image/binclf_curve_numpy.py | 49 ++------ src/anomalib/metrics/per_image/pimo.py | 25 +--- src/anomalib/metrics/per_image/pimo_numpy.py | 17 +-- src/anomalib/metrics/per_image/utils_numpy.py | 12 +- .../metrics/per_image/test_binclf_curve.py | 105 ++++------------ tests/unit/metrics/per_image/test_pimo.py | 13 -- 11 files changed, 51 insertions(+), 300 deletions(-) delete mode 100644 src/anomalib/metrics/per_image/_binclf_curve_numba.py diff --git a/pyproject.toml b/pyproject.toml index c6c79f7b53..9709c1a112 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,8 +84,7 @@ test = [ "coverage[toml]", "tox", ] -extra = ["numba>=0.58.1"] -full = ["anomalib[core,openvino,loggers,notebooks,extra]"] +full = ["anomalib[core,openvino,loggers,notebooks]"] dev = ["anomalib[full,docs,test]"] [project.scripts] diff --git a/src/anomalib/metrics/per_image/__init__.py b/src/anomalib/metrics/per_image/__init__.py index 2e34372ff7..51d27f1d49 100644 --- a/src/anomalib/metrics/per_image/__init__.py +++ b/src/anomalib/metrics/per_image/__init__.py @@ -8,7 +8,7 @@ # SPDX-License-Identifier: Apache-2.0 from .binclf_curve import per_image_binclf_curve, per_image_fpr, per_image_tpr -from .binclf_curve_numpy import BinclfAlgorithm, BinclfThreshsChoice +from .binclf_curve_numpy import BinclfThreshsChoice from .pimo import AUPIMO, PIMO, AUPIMOResult, PIMOResult, aupimo_scores, pimo_curves from .utils import ( compare_models_pairwise_ttest_rel, @@ -20,7 +20,6 @@ __all__ = [ # constants - "BinclfAlgorithm", "BinclfThreshsChoice", "StatsOutliersPolicy", "StatsRepeatedPolicy", diff --git a/src/anomalib/metrics/per_image/_binclf_curve_numba.py b/src/anomalib/metrics/per_image/_binclf_curve_numba.py deleted file mode 100644 index 3151a2faba..0000000000 --- a/src/anomalib/metrics/per_image/_binclf_curve_numba.py +++ /dev/null @@ -1,115 +0,0 @@ -"""Binary classification matrix curve (NUMBA implementation of low level functions). - -Details: `.binclf_curve`. -""" - -# Original Code -# https://github.com/jpcbertoldo/aupimo -# -# Modified -# Copyright (C) 2024 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -import numba -import numpy as np -from numpy import ndarray - - -@numba.jit(nopython=True) -def binclf_one_curve_numba(scores: ndarray, gts: ndarray, threshs: ndarray) -> ndarray: - """One binary classification matrix at each threshold (NUMBA implementation). - - This does the same as `_binclf_one_curve_python` but with numba using just-in-time compilation. - - Note: VALIDATION IS NOT DONE HERE! Make sure to validate the arguments before calling this function. - - Args: - scores (ndarray): Anomaly scores (D,). - gts (ndarray): Binary (bool) ground truth of shape (D,). - threshs (ndarray): Sequence of thresholds in ascending order (K,). - - Returns: - ndarray: Binary classification matrix curve (K, 2, 2) - - Details: `anomalib.metrics.per_image.binclf_curve_numpy.binclf_multiple_curves`. - """ - num_th = len(threshs) - - # POSITIVES - scores_pos = scores[gts] - # the sorting is very important for the algorithm to work and the speedup - scores_pos = np.sort(scores_pos) - # start counting with lowest th, so everything is predicted as positive (this variable is updated in the loop) - num_pos = current_count_tp = len(scores_pos) - - tps = np.empty((num_th,), dtype=np.int64) - - # NEGATIVES - # same thing but for the negative samples - scores_neg = scores[~gts] - scores_neg = np.sort(scores_neg) - num_neg = current_count_fp = len(scores_neg) - - fps = np.empty((num_th,), dtype=np.int64) - - # it will progressively drop the scores that are below the current th - for thidx, th in enumerate(threshs): - num_drop = 0 - num_scores = len(scores_pos) - while num_drop < num_scores and scores_pos[num_drop] < th: # ! scores_pos ! - num_drop += 1 - # --- - scores_pos = scores_pos[num_drop:] - current_count_tp -= num_drop - tps[thidx] = current_count_tp - - # same with the negatives - num_drop = 0 - num_scores = len(scores_neg) - while num_drop < num_scores and scores_neg[num_drop] < th: # ! scores_neg ! - num_drop += 1 - # --- - scores_neg = scores_neg[num_drop:] - current_count_fp -= num_drop - fps[thidx] = current_count_fp - - fns = num_pos * np.ones((num_th,), dtype=np.int64) - tps - tns = num_neg * np.ones((num_th,), dtype=np.int64) - fps - - # sequence of dimensions is (threshs, true class, predicted class) (see docstring) - return np.stack( - ( - np.stack((tns, fps), axis=-1), - np.stack((fns, tps), axis=-1), - ), - axis=-1, - ).transpose(0, 2, 1) - - -@numba.jit(nopython=True, parallel=True) -def binclf_multiple_curves_numba(scores_batch: ndarray, gts_batch: ndarray, threshs: ndarray) -> ndarray: - """Multiple binary classification matrix at each threshold (NUMBA implementation). - - This does the same as `_binclf_multiple_curves_python` but with numba, - using parallelization and just-in-time compilation. - - Note: VALIDATION IS NOT DONE HERE. Make sure to validate the arguments before calling this function. - - Args: - scores_batch (ndarray): Anomaly scores (N, D,). - gts_batch (ndarray): Binary (bool) ground truth of shape (N, D,). - threshs (ndarray): Sequence of thresholds in ascending order (K,). - - Returns: - ndarray: Binary classification matrix curves (N, K, 2, 2) - - Details: `anomalib.metrics.per_image.binclf_curve_numpy.binclf_multiple_curves`. - """ - num_imgs = scores_batch.shape[0] - num_th = len(threshs) - ret = np.empty((num_imgs, num_th, 2, 2), dtype=np.int64) - for imgidx in numba.prange(num_imgs): - scoremap = scores_batch[imgidx] - mask = gts_batch[imgidx] - ret[imgidx] = binclf_one_curve_numba(scoremap, mask, threshs) - return ret diff --git a/src/anomalib/metrics/per_image/_validate.py b/src/anomalib/metrics/per_image/_validate.py index 72f107e21e..0ebc6916e0 100644 --- a/src/anomalib/metrics/per_image/_validate.py +++ b/src/anomalib/metrics/per_image/_validate.py @@ -171,7 +171,7 @@ def is_masks(masks: ndarray) -> None: if masks.dtype.kind == "b": pass - elif masks.dtype.kind in ("i", "u"): + elif masks.dtype.kind in {"i", "u"}: masks_unique_vals = np.unique(masks) if np.any((masks_unique_vals != 0) & (masks_unique_vals != 1)): msg = ( @@ -243,7 +243,7 @@ def is_images_classes(images_classes: ndarray) -> None: if images_classes.dtype.kind == "b": pass - elif images_classes.dtype.kind in ("i", "u"): + elif images_classes.dtype.kind in {"i", "u"}: unique_vals = np.unique(images_classes) if np.any((unique_vals != 0) & (unique_vals != 1)): msg = ( diff --git a/src/anomalib/metrics/per_image/binclf_curve.py b/src/anomalib/metrics/per_image/binclf_curve.py index 1a1b614a68..4635a641c9 100644 --- a/src/anomalib/metrics/per_image/binclf_curve.py +++ b/src/anomalib/metrics/per_image/binclf_curve.py @@ -21,7 +21,7 @@ from torch import Tensor from . import _validate, binclf_curve_numpy -from .binclf_curve_numpy import BinclfAlgorithm, BinclfThreshsChoice +from .binclf_curve_numpy import BinclfThreshsChoice # =========================================== ARGS VALIDATION =========================================== @@ -47,7 +47,6 @@ def _validate_is_binclf_curves(binclf_curves: Tensor, valid_threshs: Tensor | No def per_image_binclf_curve( anomaly_maps: Tensor, masks: Tensor, - algorithm: BinclfAlgorithm | str = BinclfAlgorithm.NUMBA, threshs_choice: BinclfThreshsChoice | str = BinclfThreshsChoice.MINMAX_LINSPACE, threshs_given: Tensor | None = None, num_threshs: int | None = None, @@ -59,7 +58,6 @@ def per_image_binclf_curve( Args: anomaly_maps (Tensor): Anomaly score maps of shape (N, H, W [, D, ...]) masks (Tensor): Binary ground truth masks of shape (N, H, W [, D, ...]) - algorithm (str, optional): Algorithm to use. Defaults to ALGORITHM_NUMBA. threshs_choice (str, optional): Sequence of thresholds to use. Defaults to THRESH_SEQUENCE_MINMAX_LINSPACE. return_result_object (bool, optional): Whether to return a `PerImageBinClfCurveResult` object. Defaults to True. @@ -115,7 +113,6 @@ def per_image_binclf_curve( threshs_array, binclf_curves_array = binclf_curve_numpy.per_image_binclf_curve( anomaly_maps=anomaly_maps_array, masks=masks_array, - algorithm=algorithm, threshs_choice=threshs_choice, threshs_given=threshs_given_array, num_threshs=num_threshs, diff --git a/src/anomalib/metrics/per_image/binclf_curve_numpy.py b/src/anomalib/metrics/per_image/binclf_curve_numpy.py index 621932baeb..ae444735b6 100644 --- a/src/anomalib/metrics/per_image/binclf_curve_numpy.py +++ b/src/anomalib/metrics/per_image/binclf_curve_numpy.py @@ -20,17 +20,6 @@ import numpy as np from numpy import ndarray -try: - import numba # noqa: F401 -except ImportError: - HAS_NUMBA = False -else: - HAS_NUMBA = True - - -if HAS_NUMBA: - from . import _binclf_curve_numba - from . import _validate logger = logging.getLogger(__name__) @@ -38,13 +27,6 @@ # =========================================== CONSTANTS =========================================== -class BinclfAlgorithm(Enum): - """Algorithm to use (relates to the low-level implementation).""" - - PYTHON: str = "python" - NUMBA: str = "numba" - - class BinclfThreshsChoice(Enum): """Sequence of thresholds to use.""" @@ -95,8 +77,8 @@ def _validate_is_gts_batch(gts_batch: ndarray) -> None: # =========================================== PYTHON VERSION =========================================== -def _binclf_one_curve_python(scores: ndarray, gts: ndarray, threshs: ndarray) -> ndarray: - """One binary classification matrix at each threshold (PYTHON implementation). +def _binclf_one_curve(scores: ndarray, gts: ndarray, threshs: ndarray) -> ndarray: + """One binary classification matrix at each threshold. In the case where the thresholds are given (i.e. not considering all possible thresholds based on the scores), this weird-looking function is faster than the two options in `torchmetrics` on the CPU: @@ -167,10 +149,10 @@ def score_less_than_thresh(score: float, thresh: float) -> bool: ).transpose(0, 2, 1) -_binclf_multiple_curves_python = np.vectorize(_binclf_one_curve_python, signature="(n),(n),(k)->(k,2,2)") -_binclf_multiple_curves_python.__doc__ = """ -Multiple binary classification matrix at each threshold (PYTHON implementation). -vectorized version of `_binclf_one_curve_python` (see above) +_binclf_multiple_curves = np.vectorize(_binclf_one_curve, signature="(n),(n),(k)->(k,2,2)") +_binclf_multiple_curves.__doc__ = """ +Multiple binary classification matrix at each threshold. +vectorized version of `_binclf_one_curve` (see above) """ # =========================================== INTERFACE =========================================== @@ -180,7 +162,6 @@ def binclf_multiple_curves( scores_batch: ndarray, gts_batch: ndarray, threshs: ndarray, - algorithm: BinclfAlgorithm | str = BinclfAlgorithm.NUMBA.value, ) -> ndarray: """Multiple binary classification matrix (per-instance scope) at each threshold (shared). @@ -220,23 +201,12 @@ def binclf_multiple_curves( Thresholds are sorted in ascending order. """ - algorithm = BinclfAlgorithm(algorithm) _validate_is_scores_batch(scores_batch) _validate_is_gts_batch(gts_batch) _validate.is_same_shape(scores_batch, gts_batch) _validate.is_threshs(threshs) - if algorithm == BinclfAlgorithm.NUMBA: - if HAS_NUMBA: - return _binclf_curve_numba.binclf_multiple_curves_numba(scores_batch, gts_batch, threshs) - - logger.warning( - f"Algorithm '{BinclfAlgorithm.NUMBA.value}' was selected, but Numba is not installed. " - f"Falling back to '{BinclfAlgorithm.PYTHON.value}' implementation.", - "Notice that the performance will be slower. Consider installing Numba for faster computation.", - ) - - return _binclf_multiple_curves_python(scores_batch, gts_batch, threshs) + return _binclf_multiple_curves(scores_batch, gts_batch, threshs) # ========================================= PER-IMAGE BINCLF CURVE ========================================= @@ -258,7 +228,6 @@ def _get_threshs_minmax_linspace(anomaly_maps: ndarray, num_threshs: int) -> nda def per_image_binclf_curve( anomaly_maps: ndarray, masks: ndarray, - algorithm: BinclfAlgorithm | str = BinclfAlgorithm.NUMBA.value, threshs_choice: BinclfThreshsChoice | str = BinclfThreshsChoice.MINMAX_LINSPACE.value, threshs_given: ndarray | None = None, num_threshs: int | None = None, @@ -268,7 +237,6 @@ def per_image_binclf_curve( Args: anomaly_maps (ndarray): Anomaly score maps of shape (N, H, W) masks (ndarray): Binary ground truth masks of shape (N, H, W) - algorithm (str, optional): Algorithm to use. Defaults to ALGORITHM_NUMBA. threshs_choice (str, optional): Sequence of thresholds to use. Defaults to THRESH_SEQUENCE_MINMAX_LINSPACE. # # `threshs_choice`-dependent arguments @@ -308,7 +276,6 @@ def per_image_binclf_curve( Thresholds are sorted in ascending order. """ - BinclfAlgorithm(algorithm) threshs_choice = BinclfThreshsChoice(threshs_choice) _validate.is_anomaly_maps(anomaly_maps) _validate.is_masks(masks) @@ -350,7 +317,7 @@ def per_image_binclf_curve( scores_batch = anomaly_maps.reshape(anomaly_maps.shape[0], -1) gts_batch = masks.reshape(masks.shape[0], -1).astype(bool) # make sure it is boolean - binclf_curves = binclf_multiple_curves(scores_batch, gts_batch, threshs, algorithm=algorithm) + binclf_curves = binclf_multiple_curves(scores_batch, gts_batch, threshs) num_images = anomaly_maps.shape[0] diff --git a/src/anomalib/metrics/per_image/pimo.py b/src/anomalib/metrics/per_image/pimo.py index f29056e973..6c329f211a 100644 --- a/src/anomalib/metrics/per_image/pimo.py +++ b/src/anomalib/metrics/per_image/pimo.py @@ -53,7 +53,6 @@ from anomalib.data.utils.path import validate_path from . import _validate, pimo_numpy, utils -from .binclf_curve_numpy import BinclfAlgorithm from .utils import StatsOutliersPolicy, StatsRepeatedPolicy logger = logging.getLogger(__name__) @@ -544,7 +543,6 @@ def pimo_curves( anomaly_maps: Tensor, masks: Tensor, num_threshs: int, - binclf_algorithm: BinclfAlgorithm | str = BinclfAlgorithm.NUMBA.value, paths: list[str] | None = None, ) -> PIMOResult: """Compute the Per-IMage Overlap (PIMO, pronounced pee-mo) curves. @@ -569,7 +567,6 @@ def pimo_curves( anomaly_maps: floating point anomaly score maps of shape (N, H, W) masks: binary (bool or int) ground truth masks of shape (N, H, W) num_threshs: number of thresholds to compute (K) - binclf_algorithm: algorithm to compute the binary classifier curve (see `binclf_curve_numpy.Algorithm`) paths: paths to the source images to which the PIMO curves correspond. Default: None. Returns: @@ -589,7 +586,6 @@ def pimo_curves( anomaly_maps_array, masks_array, num_threshs, - binclf_algorithm=binclf_algorithm, ) # _ is `image_classes` -- not needed here because it's a property in the result object @@ -616,7 +612,6 @@ def aupimo_scores( anomaly_maps: Tensor, masks: Tensor, num_threshs: int = 300_000, - binclf_algorithm: BinclfAlgorithm | str = BinclfAlgorithm.NUMBA.value, fpr_bounds: tuple[float, float] = (1e-5, 1e-4), force: bool = False, paths: list[str] | None = None, @@ -642,7 +637,6 @@ def aupimo_scores( anomaly_maps: floating point anomaly score maps of shape (N, H, W) masks: binary (bool or int) ground truth masks of shape (N, H, W) num_threshs: number of thresholds to compute (K) - binclf_algorithm: algorithm to compute the binary classifier curve (see `binclf_curve_numpy.Algorithm`) fpr_bounds: lower and upper bounds of the FPR integration range force: whether to force the computation despite bad conditions paths: paths to the source images to which the AUPIMO scores correspond. @@ -662,11 +656,9 @@ def aupimo_scores( anomaly_maps_array, masks_array, num_threshs, - binclf_algorithm=binclf_algorithm, fpr_bounds=fpr_bounds, force=force, ) - # tensors are build with `torch.from_numpy` and so the returned tensors # will share the same memory as the numpy arrays device = anomaly_maps.device @@ -750,7 +742,7 @@ def _is_empty(self) -> bool: @property def num_images(self) -> int: """Number of images.""" - return sum([am.shape[0] for am in self.anomaly_maps]) + return sum(am.shape[0] for am in self.anomaly_maps) @property def image_classes(self) -> Tensor: @@ -760,13 +752,11 @@ def image_classes(self) -> Tensor: def __init__( self, num_threshs: int, - binclf_algorithm: BinclfAlgorithm | str = BinclfAlgorithm.NUMBA.value, ) -> None: """Per-Image Overlap (PIMO) curve. Args: num_threshs: number of thresholds used to compute the PIMO curve (K) - binclf_algorithm: algorithm to compute the binary classification curve (see `binclf_curve_numpy.Algorithm`) """ super().__init__() @@ -781,9 +771,6 @@ def __init__( _validate.is_num_threshs_gte2(num_threshs) self.num_threshs = num_threshs - # validate binclf_algorithm and get string - self.binclf_algorithm = BinclfAlgorithm(binclf_algorithm).value - self.add_state("anomaly_maps", default=[], dist_reduce_fx="cat") self.add_state("masks", default=[], dist_reduce_fx="cat") @@ -817,7 +804,6 @@ def compute(self) -> PIMOResult: anomaly_maps, masks, self.num_threshs, - binclf_algorithm=self.binclf_algorithm, ) @@ -845,7 +831,6 @@ class AUPIMO(PIMO): Args: num_threshs: number of thresholds to compute (K) - binclf_algorithm: algorithm to compute the binary classifier curve (see `binclf_curve_numpy.Algorithm`) fpr_bounds: lower and upper bounds of the FPR integration range force: whether to force the computation despite bad conditions @@ -897,7 +882,6 @@ def __repr__(self) -> str: def __init__( self, num_threshs: int = 300_000, - binclf_algorithm: BinclfAlgorithm | str = BinclfAlgorithm.NUMBA.value, fpr_bounds: tuple[float, float] = (1e-5, 1e-4), return_average: bool = True, force: bool = False, @@ -906,15 +890,11 @@ def __init__( Args: num_threshs: [passed to parent `PIMO`] number of thresholds used to compute the PIMO curve - binclf_algorithm: [passed to parent `PIMO`] algorithm to compute the binary classification curve fpr_bounds: lower and upper bounds of the FPR integration range return_average: if True, return the average AUPIMO score; if False, return all the individual AUPIMO scores force: if True, force the computation of the AUPIMO scores even in bad conditions (e.g. few points) """ - super().__init__( - num_threshs=num_threshs, - binclf_algorithm=binclf_algorithm, - ) + super().__init__(num_threshs=num_threshs) # other validations are done in PIMO.__init__() @@ -945,7 +925,6 @@ def compute(self, force: bool | None = None) -> tuple[PIMOResult, AUPIMOResult]: anomaly_maps, masks, self.num_threshs, - binclf_algorithm=self.binclf_algorithm, fpr_bounds=self.fpr_bounds, force=force, ) diff --git a/src/anomalib/metrics/per_image/pimo_numpy.py b/src/anomalib/metrics/per_image/pimo_numpy.py index 8b1f56f7ff..d979cf1a53 100644 --- a/src/anomalib/metrics/per_image/pimo_numpy.py +++ b/src/anomalib/metrics/per_image/pimo_numpy.py @@ -17,7 +17,7 @@ from numpy import ndarray from . import _validate, binclf_curve_numpy -from .binclf_curve_numpy import BinclfAlgorithm, BinclfThreshsChoice +from .binclf_curve_numpy import BinclfThreshsChoice logger = logging.getLogger(__name__) @@ -72,7 +72,6 @@ def pimo_curves( anomaly_maps: ndarray, masks: ndarray, num_threshs: int, - binclf_algorithm: BinclfAlgorithm | str = BinclfAlgorithm.NUMBA.value, ) -> tuple[ndarray, ndarray, ndarray, ndarray]: """Compute the Per-IMage Overlap (PIMO, pronounced pee-mo) curves. @@ -92,7 +91,6 @@ def pimo_curves( anomaly_maps: floating point anomaly score maps of shape (N, H, W) masks: binary (bool or int) ground truth masks of shape (N, H, W) num_threshs: number of thresholds to compute (K) - binclf_algorithm: algorithm to compute the binary classifier curve (see `binclf_curve_numpy.Algorithm`) Returns: tuple[ndarray, ndarray, ndarray, ndarray]: @@ -102,11 +100,10 @@ def pimo_curves( [3] image classes of shape (N,) with values 0 (normal) or 1 (anomalous) """ # validate the strings are valid - BinclfAlgorithm(binclf_algorithm) _validate.is_num_threshs_gte2(num_threshs) - _validate.is_anomaly_maps(anomaly_maps) - _validate.is_masks(masks) - _validate.is_same_shape(anomaly_maps, masks) + _validate.is_anomaly_maps(anomaly_maps) # redundant + _validate.is_masks(masks) # redundant + _validate.is_same_shape(anomaly_maps, masks) # redundant _validate_has_at_least_one_anomalous_image(masks) _validate_has_at_least_one_normal_image(masks) @@ -126,7 +123,6 @@ def pimo_curves( threshs, binclf_curves = binclf_curve_numpy.per_image_binclf_curve( anomaly_maps=anomaly_maps, masks=masks, - algorithm=binclf_algorithm, threshs_choice=BinclfThreshsChoice.GIVEN.value, threshs_given=threshs, num_threshs=None, @@ -160,7 +156,6 @@ def aupimo_scores( anomaly_maps: ndarray, masks: ndarray, num_threshs: int = 300_000, - binclf_algorithm: BinclfAlgorithm | str = BinclfAlgorithm.NUMBA, fpr_bounds: tuple[float, float] = (1e-5, 1e-4), force: bool = False, ) -> tuple[ndarray, ndarray, ndarray, ndarray, ndarray, int]: @@ -181,7 +176,6 @@ def aupimo_scores( anomaly_maps: floating point anomaly score maps of shape (N, H, W) masks: binary (bool or int) ground truth masks of shape (N, H, W) num_threshs: number of thresholds to compute (K) - binclf_algorithm: algorithm to compute the binary classifier curve (see `binclf_curve_numpy.Algorithm`) fpr_bounds: lower and upper bounds of the FPR integration range force: whether to force the computation despite bad conditions @@ -201,7 +195,6 @@ def aupimo_scores( anomaly_maps=anomaly_maps, masks=masks, num_threshs=num_threshs, - binclf_algorithm=binclf_algorithm, ) try: _validate.is_threshs(threshs) @@ -303,7 +296,7 @@ def aupimo_scores( "Try increasing `num_threshs`.", ) - aucs: ndarray = np.trapezoid(per_image_tprs_bounded, x=shared_fpr_bounded_log, axis=1) + aucs: ndarray = np.trapz(per_image_tprs_bounded, x=shared_fpr_bounded_log, axis=1) # noqa: NPY201 deprecated in Numpy 2.0 # normalize, then clip(0, 1) makes sure that the values are in [0, 1] in case of numerical errors normalization_factor = aupimo_normalizing_factor(fpr_bounds) diff --git a/src/anomalib/metrics/per_image/utils_numpy.py b/src/anomalib/metrics/per_image/utils_numpy.py index 736780831c..7eb5413346 100644 --- a/src/anomalib/metrics/per_image/utils_numpy.py +++ b/src/anomalib/metrics/per_image/utils_numpy.py @@ -66,7 +66,7 @@ def _validate_is_image_class(image_class: int) -> None: msg = f"Expected image class to be an int (0 for 'normal', 1 for 'anomalous'), but got {type(image_class)}." raise TypeError(msg) - if image_class not in (0, 1): + if image_class not in {0, 1}: msg = f"Expected image class to be either 0 for 'normal' or 1 for 'anomalous', but got {image_class}." raise ValueError(msg) @@ -236,7 +236,7 @@ def per_image_scores_stats( boxplot_stats = mpl.cbook.boxplot_stats(per_image_scores)[0] # [0] is for the only boxplot # remove unnecessary keys - boxplot_stats = {name: value for name, value in boxplot_stats.items() if name not in ("iqr", "cilo", "cihi")} + boxplot_stats = {name: value for name, value in boxplot_stats.items() if name not in {"iqr", "cilo", "cihi"}} # unroll `fliers` (outliers), remove unnecessary ones according to `outliers_policy`, # then add them to `boxplot_stats` with unique keys @@ -244,13 +244,13 @@ def per_image_scores_stats( outliers_lo = outliers[outliers < boxplot_stats["med"]] outliers_hi = outliers[outliers > boxplot_stats["med"]] - if outliers_policy in (StatsOutliersPolicy.HI, StatsOutliersPolicy.BOTH): + if outliers_policy in {StatsOutliersPolicy.HI, StatsOutliersPolicy.BOTH}: boxplot_stats = { **boxplot_stats, **{f"outhi_{idx:06}": value for idx, value in enumerate(outliers_hi)}, } - if outliers_policy in (StatsOutliersPolicy.LO, StatsOutliersPolicy.BOTH): + if outliers_policy in {StatsOutliersPolicy.LO, StatsOutliersPolicy.BOTH}: boxplot_stats = { **boxplot_stats, **{f"outlo_{idx:06}": value for idx, value in enumerate(outliers_lo)}, @@ -378,7 +378,7 @@ def compare_models_pairwise_ttest_rel( values_j, alternative=alternative, ).pvalue - confidences[(model_i, model_j)] = 1.0 - float(pvalue) + confidences[model_i, model_j] = 1.0 - float(pvalue) return models_ordered, confidences @@ -476,6 +476,6 @@ def compare_models_pairwise_wilcoxon( pvalue = 1.0 else: pvalue = scipy.stats.wilcoxon(diff, alternative=alternative).pvalue - confidences[(model_i, model_j)] = 1.0 - float(pvalue) + confidences[model_i, model_j] = 1.0 - float(pvalue) return models_ordered, confidences diff --git a/tests/unit/metrics/per_image/test_binclf_curve.py b/tests/unit/metrics/per_image/test_binclf_curve.py index 6b0499bf9a..62112bc257 100644 --- a/tests/unit/metrics/per_image/test_binclf_curve.py +++ b/tests/unit/metrics/per_image/test_binclf_curve.py @@ -1,4 +1,4 @@ -"""Tests for per-image binary classification curves using numpy and numba versions.""" +"""Tests for per-image binary classification curves using numpy version.""" # Original Code # https://github.com/jpcbertoldo/aupimo @@ -16,10 +16,6 @@ from torch import Tensor from anomalib.metrics.per_image import binclf_curve, binclf_curve_numpy -from anomalib.metrics.per_image.binclf_curve_numpy import HAS_NUMBA - -if HAS_NUMBA: - from anomalib.metrics.per_image import _binclf_curve_numba def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: @@ -98,7 +94,7 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: axis=0, ).astype(int) - if metafunc.function is test__binclf_one_curve_python or metafunc.function is test__binclf_one_curve_numba: + if metafunc.function is test__binclf_one_curve: metafunc.parametrize( argnames=("pred", "gt", "threshs", "expected"), argvalues=[ @@ -118,10 +114,7 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: binclf_curves_threshs_too_high = np.stack([expected_anom_threshs_too_high, expected_norm_threshs_too_high], axis=0) binclf_curves_threshs_too_low = np.stack([expected_anom_threshs_too_low, expected_norm_threshs_too_low], axis=0) - if ( - metafunc.function is test__binclf_multiple_curves_python - or metafunc.function is test__binclf_multiple_curves_numba - ): + if metafunc.function is test__binclf_multiple_curves: metafunc.parametrize( argnames=("preds", "gts", "threshs", "expecteds"), argvalues=[ @@ -144,39 +137,30 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: (10 * preds, gts, 10 * threshs, binclf_curves), ], ) - metafunc.parametrize( - argnames=("algorithm",), - argvalues=[ - ("python",), - ("numba",), - ], - ) if metafunc.function is test_binclf_multiple_curves_validations: metafunc.parametrize( argnames=("args", "kwargs", "exception"), argvalues=[ # `scores` and `gts` must be 2D - ([preds.reshape(2, 2, 2), gts, threshs], {"algorithm": "numba"}, ValueError), - ([preds, gts.flatten(), threshs], {"algorithm": "numba"}, ValueError), + ([preds.reshape(2, 2, 2), gts, threshs], {}, ValueError), + ([preds, gts.flatten(), threshs], {}, ValueError), # `threshs` must be 1D - ([preds, gts, threshs.reshape(2, 2)], {"algorithm": "numba"}, ValueError), + ([preds, gts, threshs.reshape(2, 2)], {}, ValueError), # `scores` and `gts` must have the same shape - ([preds, gts[:1], threshs], {"algorithm": "numba"}, ValueError), - ([preds[:, :2], gts, threshs], {"algorithm": "numba"}, ValueError), + ([preds, gts[:1], threshs], {}, ValueError), + ([preds[:, :2], gts, threshs], {}, ValueError), # `scores` be of type float - ([preds.astype(int), gts, threshs], {"algorithm": "numba"}, TypeError), + ([preds.astype(int), gts, threshs], {}, TypeError), # `gts` be of type bool - ([preds, gts.astype(int), threshs], {"algorithm": "numba"}, TypeError), + ([preds, gts.astype(int), threshs], {}, TypeError), # `threshs` be of type float - ([preds, gts, threshs.astype(int)], {"algorithm": "numba"}, TypeError), + ([preds, gts, threshs.astype(int)], {}, TypeError), # `threshs` must be sorted in ascending order - ([preds, gts, np.flip(threshs)], {"algorithm": "numba"}, ValueError), - ([preds, gts, np.concatenate([threshs[-2:], threshs[:2]])], {"algorithm": "numba"}, ValueError), + ([preds, gts, np.flip(threshs)], {}, ValueError), + ([preds, gts, np.concatenate([threshs[-2:], threshs[:2]])], {}, ValueError), # `threshs` must be unique - ([preds, gts, np.sort(np.concatenate([threshs, threshs]))], {"algorithm": "numba"}, ValueError), - # invalid `algorithm` - ([preds, gts, threshs], {"algorithm": "blurp"}, ValueError), + ([preds, gts, np.sort(np.concatenate([threshs, threshs]))], {}, ValueError), ], ) @@ -267,15 +251,6 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: ], ) - if metafunc.function is test_per_image_binclf_curve_numpy or metafunc.function is test_per_image_binclf_curve_torch: - metafunc.parametrize( - argnames=("algorithm",), - argvalues=[ - ("python",), - ("numba",), - ], - ) - if metafunc.function is test_per_image_binclf_curve_numpy_validations: metafunc.parametrize( argnames=("args", "exception"), @@ -297,10 +272,8 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: metafunc.parametrize( argnames=("kwargs",), argvalues=[ - ({"algorithm": "numba", "threshs_choice": "given", "threshs_given": threshs, "num_threshs": None},), ( { - "algorithm": "python", "threshs_choice": "minmax-linspace", "threshs_given": None, "num_threshs": len(threshs), @@ -317,7 +290,7 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: # invalid `threshs_choice` ( [preds, gts], - {"algorithm": "glfrb", "threshs_choice": "given", "threshs_given": threshs, "num_threshs": None}, + {"threshs_choice": "glfrb", "threshs_given": threshs, "num_threshs": None}, ValueError, ), ], @@ -345,43 +318,21 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: # LOW-LEVEL FUNCTIONS (PYTHON) -def test__binclf_one_curve_python(pred: ndarray, gt: ndarray, threshs: ndarray, expected: ndarray) -> None: - """Test if `_binclf_one_curve_python()` returns the expected values.""" - computed = binclf_curve_numpy._binclf_one_curve_python(pred, gt, threshs) +def test__binclf_one_curve(pred: ndarray, gt: ndarray, threshs: ndarray, expected: ndarray) -> None: + """Test if `_binclf_one_curve()` returns the expected values.""" + computed = binclf_curve_numpy._binclf_one_curve(pred, gt, threshs) assert computed.shape == (threshs.size, 2, 2) assert (computed == expected).all() -def test__binclf_multiple_curves_python( +def test__binclf_multiple_curves( preds: ndarray, gts: ndarray, threshs: ndarray, expecteds: ndarray, ) -> None: - """Test if `_binclf_multiple_curves_python()` returns the expected values.""" - computed = binclf_curve_numpy._binclf_multiple_curves_python(preds, gts, threshs) - assert computed.shape == (preds.shape[0], threshs.size, 2, 2) - assert (computed == expecteds).all() - - -# ================================================================================================== -# LOW-LEVEL FUNCTIONS (NUMBA) - - -def test__binclf_one_curve_numba(pred: ndarray, gt: ndarray, threshs: ndarray, expected: ndarray) -> None: - """Test if `_binclf_one_curve_numba()` returns the expected values.""" - if not HAS_NUMBA: - pytest.skip("Numba is not available.") - computed = _binclf_curve_numba.binclf_one_curve_numba(pred, gt, threshs) - assert computed.shape == (threshs.size, 2, 2) - assert (computed == expected).all() - - -def test__binclf_multiple_curves_numba(preds: ndarray, gts: ndarray, threshs: ndarray, expecteds: ndarray) -> None: - """Test if `_binclf_multiple_curves_python()` returns the expected values.""" - if not HAS_NUMBA: - pytest.skip("Numba is not available.") - computed = _binclf_curve_numba.binclf_multiple_curves_numba(preds, gts, threshs) + """Test if `_binclf_multiple_curves()` returns the expected values.""" + computed = binclf_curve_numpy.binclf_multiple_curves(preds, gts, threshs) assert computed.shape == (preds.shape[0], threshs.size, 2, 2) assert (computed == expecteds).all() @@ -395,33 +346,31 @@ def test_binclf_multiple_curves( gts: ndarray, threshs: ndarray, expected_binclf_curves: ndarray, - algorithm: str, ) -> None: """Test if `binclf_multiple_curves()` returns the expected values.""" computed = binclf_curve_numpy.binclf_multiple_curves( preds, gts, threshs, - algorithm=algorithm, ) assert computed.shape == expected_binclf_curves.shape assert (computed == expected_binclf_curves).all() # it's ok to have the threhsholds beyond the range of the preds - binclf_curve_numpy.binclf_multiple_curves(preds, gts, 2 * threshs, algorithm=algorithm) + binclf_curve_numpy.binclf_multiple_curves(preds, gts, 2 * threshs) # or inside the bounds without reaching them - binclf_curve_numpy.binclf_multiple_curves(preds, gts, 0.5 * threshs, algorithm=algorithm) + binclf_curve_numpy.binclf_multiple_curves(preds, gts, 0.5 * threshs) # it's also ok to have more threshs than unique values in the preds # add the values in between the threshs threshs_unncessary = 0.5 * (threshs[:-1] + threshs[1:]) threshs_unncessary = np.concatenate([threshs_unncessary, threshs]) threshs_unncessary = np.sort(threshs_unncessary) - binclf_curve_numpy.binclf_multiple_curves(preds, gts, threshs_unncessary, algorithm=algorithm) + binclf_curve_numpy.binclf_multiple_curves(preds, gts, threshs_unncessary) # or less - binclf_curve_numpy.binclf_multiple_curves(preds, gts, threshs[1:3], algorithm=algorithm) + binclf_curve_numpy.binclf_multiple_curves(preds, gts, threshs[1:3]) def test_binclf_multiple_curves_validations(args: list, kwargs: dict, exception: Exception) -> None: @@ -433,7 +382,6 @@ def test_binclf_multiple_curves_validations(args: list, kwargs: dict, exception: def test_per_image_binclf_curve_numpy( anomaly_maps: ndarray, masks: ndarray, - algorithm: str, threshs_choice: str, threshs_given: ndarray | None, num_threshs: int | None, @@ -444,7 +392,6 @@ def test_per_image_binclf_curve_numpy( computed_threshs, computed_binclf_curves = binclf_curve_numpy.per_image_binclf_curve( anomaly_maps, masks, - algorithm=algorithm, threshs_choice=threshs_choice, threshs_given=threshs_given, num_threshs=num_threshs, @@ -491,7 +438,6 @@ def test_rate_metrics_numpy(binclf_curves: ndarray, expected_fprs: ndarray, expe def test_per_image_binclf_curve_torch( anomaly_maps: Tensor, masks: Tensor, - algorithm: str, threshs_choice: str, threshs_given: Tensor | None, num_threshs: int | None, @@ -502,7 +448,6 @@ def test_per_image_binclf_curve_torch( computed_threshs, computed_binclf_curves = binclf_curve.per_image_binclf_curve( anomaly_maps, masks, - algorithm=algorithm, threshs_choice=threshs_choice, threshs_given=threshs_given, num_threshs=num_threshs, diff --git a/tests/unit/metrics/per_image/test_pimo.py b/tests/unit/metrics/per_image/test_pimo.py index ce30a13542..9e818346fb 100644 --- a/tests/unit/metrics/per_image/test_pimo.py +++ b/tests/unit/metrics/per_image/test_pimo.py @@ -252,7 +252,6 @@ def test_pimo_numpy( anomaly_maps, masks, num_threshs=7, - binclf_algorithm="numba", ) _do_test_pimo_outputs( threshs, @@ -297,14 +296,12 @@ def do_assertions(pimoresult: PIMOResult) -> None: anomaly_maps, masks, num_threshs=7, - binclf_algorithm="numba", ) do_assertions(pimoresult) # metric interface metric = pimo.PIMO( num_threshs=7, - binclf_algorithm="numba", ) metric.update(anomaly_maps, masks) pimoresult = metric.compute() @@ -361,7 +358,6 @@ def test_aupimo_values_numpy( anomaly_maps, masks, num_threshs=7, - binclf_algorithm="numba", fpr_bounds=fpr_bounds, force=True, ) @@ -428,7 +424,6 @@ def do_assertions(pimoresult: PIMOResult, aupimoresult: AUPIMOResult) -> None: anomaly_maps, masks, num_threshs=7, - binclf_algorithm="numba", fpr_bounds=fpr_bounds, force=True, ) @@ -437,7 +432,6 @@ def do_assertions(pimoresult: PIMOResult, aupimoresult: AUPIMOResult) -> None: # metric interface metric = pimo.AUPIMO( num_threshs=7, - binclf_algorithm="numba", fpr_bounds=fpr_bounds, return_average=False, force=True, @@ -449,7 +443,6 @@ def do_assertions(pimoresult: PIMOResult, aupimoresult: AUPIMOResult) -> None: # metric interface metric = pimo.AUPIMO( num_threshs=7, - binclf_algorithm="numba", fpr_bounds=fpr_bounds, return_average=True, # only return the average AUPIMO force=True, @@ -474,7 +467,6 @@ def test_aupimo_edge( anomaly_maps, masks, num_threshs=10, - binclf_algorithm="numba", force=False, **fpr_bounds, ) @@ -484,7 +476,6 @@ def test_aupimo_edge( anomaly_maps, masks, num_threshs=10, - binclf_algorithm="numba", force=True, **fpr_bounds, ) @@ -494,8 +485,6 @@ def test_aupimo_edge( pimo_numpy.aupimo_scores( anomaly_maps * rng.uniform(1.0, 1.1, size=anomaly_maps.shape), masks, - # num_threshs=, - binclf_algorithm="numba", force=False, **fpr_bounds, ) @@ -515,7 +504,6 @@ def test_pimoresult_object( anomaly_maps, masks, num_threshs=7, - binclf_algorithm="numba", **optional_kwargs, ) @@ -560,7 +548,6 @@ def test_aupimoresult_object( anomaly_maps, masks, num_threshs=7, - binclf_algorithm="numba", fpr_bounds=(1e-5, 1e-4), force=True, **optional_kwargs, From 36d37f312c83d13ea3045d779e49d83c2bb27cef Mon Sep 17 00:00:00 2001 From: Ashwin Vaidya Date: Mon, 23 Sep 2024 09:27:43 +0200 Subject: [PATCH 2/3] fix pre-commit checks Signed-off-by: Ashwin Vaidya --- tests/unit/data/utils/test_path.py | 3 ++- tests/unit/metrics/per_image/test_utils.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/unit/data/utils/test_path.py b/tests/unit/data/utils/test_path.py index 5a4b8fee45..f1764b7373 100644 --- a/tests/unit/data/utils/test_path.py +++ b/tests/unit/data/utils/test_path.py @@ -77,7 +77,8 @@ def test_no_read_execute_permission() -> None: with pytest.raises(PermissionError, match=r"Read or execute permissions denied for the path:*"): validate_path(tmp_dir, base_dir=Path(tmp_dir)) - def test_file_wrongsuffix(self) -> None: + @staticmethod + def test_file_wrongsuffix() -> None: """Test ``validate_path`` raises ValueError for a file with wrong suffix.""" with pytest.raises(ValueError, match="Path extension is not accepted."): validate_path("file.png", should_exist=False, accepted_extensions=(".json", ".txt")) diff --git a/tests/unit/metrics/per_image/test_utils.py b/tests/unit/metrics/per_image/test_utils.py index 0e712b6584..0c894bf56f 100644 --- a/tests/unit/metrics/per_image/test_utils.py +++ b/tests/unit/metrics/per_image/test_utils.py @@ -88,7 +88,7 @@ def assert_statsdict_stuff(statdic: dict, max_image_idx: int) -> None: """Assert stuff about a `statdic`.""" assert "stat_name" in statdic stat_name = statdic["stat_name"] - assert stat_name in ("mean", "med", "q1", "q3", "whishi", "whislo") or stat_name.startswith( + assert stat_name in {"mean", "med", "q1", "q3", "whishi", "whislo"} or stat_name.startswith( ("outlo_", "outhi_"), ) assert "stat_value" in statdic From dbd9627ddc009a0f9ae9228ba8b76844f9ce58a2 Mon Sep 17 00:00:00 2001 From: Ashwin Vaidya Date: Mon, 23 Sep 2024 14:13:42 +0200 Subject: [PATCH 3/3] remove all unused methods Signed-off-by: Ashwin Vaidya --- src/anomalib/metrics/per_image/__init__.py | 4 - .../metrics/per_image/binclf_curve.py | 170 ------------------ src/anomalib/metrics/per_image/pimo.py | 168 +---------------- src/anomalib/metrics/per_image/pimo_numpy.py | 29 --- src/anomalib/metrics/per_image/utils.py | 4 +- src/anomalib/metrics/per_image/utils_numpy.py | 16 +- .../metrics/per_image/test_binclf_curve.py | 75 +------- tests/unit/metrics/per_image/test_pimo.py | 112 ------------ tests/unit/metrics/per_image/test_utils.py | 4 +- 9 files changed, 14 insertions(+), 568 deletions(-) delete mode 100644 src/anomalib/metrics/per_image/binclf_curve.py diff --git a/src/anomalib/metrics/per_image/__init__.py b/src/anomalib/metrics/per_image/__init__.py index 51d27f1d49..b98ea9fae6 100644 --- a/src/anomalib/metrics/per_image/__init__.py +++ b/src/anomalib/metrics/per_image/__init__.py @@ -7,7 +7,6 @@ # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -from .binclf_curve import per_image_binclf_curve, per_image_fpr, per_image_tpr from .binclf_curve_numpy import BinclfThreshsChoice from .pimo import AUPIMO, PIMO, AUPIMOResult, PIMOResult, aupimo_scores, pimo_curves from .utils import ( @@ -27,9 +26,6 @@ "PIMOResult", "AUPIMOResult", # functional interfaces - "per_image_binclf_curve", - "per_image_fpr", - "per_image_tpr", "pimo_curves", "aupimo_scores", # torchmetrics interfaces diff --git a/src/anomalib/metrics/per_image/binclf_curve.py b/src/anomalib/metrics/per_image/binclf_curve.py deleted file mode 100644 index 4635a641c9..0000000000 --- a/src/anomalib/metrics/per_image/binclf_curve.py +++ /dev/null @@ -1,170 +0,0 @@ -"""Binary classification curve (torch interface). - -This module implements torch interfaces to access the numpy code in `binclf_curve_numpy.py`. - -Details: `anomalib.metrics.per_image.binclf_curve_numpy.binclf_multiple_curves`. - -Tensors are build with `torch.from_numpy` and so the returned tensors will share the same memory as the numpy arrays. - -Validations will preferably happen in ndarray so the numpy code can be reused without torch, -so often times the Tensor arguments will be converted to ndarray and then validated. -""" - -# Original Code -# https://github.com/jpcbertoldo/aupimo -# -# Modified -# Copyright (C) 2024 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -import torch -from torch import Tensor - -from . import _validate, binclf_curve_numpy -from .binclf_curve_numpy import BinclfThreshsChoice - -# =========================================== ARGS VALIDATION =========================================== - - -def _validate_is_threshs(threshs: Tensor) -> None: - _validate.is_tensor(threshs, argname="threshs") - _validate.is_threshs(threshs.numpy()) - - -def _validate_is_binclf_curves(binclf_curves: Tensor, valid_threshs: Tensor | None = None) -> None: - _validate.is_tensor(binclf_curves, argname="binclf_curves") - if valid_threshs is not None: - _validate_is_threshs(valid_threshs) - _validate.is_binclf_curves( - binclf_curves.detach().cpu().numpy(), - valid_threshs=valid_threshs.numpy() if valid_threshs is not None else None, - ) - - -# =========================================== FUNCTIONAL =========================================== - - -def per_image_binclf_curve( - anomaly_maps: Tensor, - masks: Tensor, - threshs_choice: BinclfThreshsChoice | str = BinclfThreshsChoice.MINMAX_LINSPACE, - threshs_given: Tensor | None = None, - num_threshs: int | None = None, -) -> tuple[Tensor, Tensor]: - """Compute the binary classification matrix of each image in the batch for multiple thresholds (shared). - - Note: tensors are converted to numpy arrays and then converted back to tensors (same device as `anomaly_maps`). - - Args: - anomaly_maps (Tensor): Anomaly score maps of shape (N, H, W [, D, ...]) - masks (Tensor): Binary ground truth masks of shape (N, H, W [, D, ...]) - threshs_choice (str, optional): Sequence of thresholds to use. Defaults to THRESH_SEQUENCE_MINMAX_LINSPACE. - return_result_object (bool, optional): Whether to return a `PerImageBinClfCurveResult` object. Defaults to True. - - *** `threshs_choice`-dependent arguments *** - - THRESH_SEQUENCE_GIVEN - threshs_given (Tensor, optional): Sequence of thresholds to use. - - THRESH_SEQUENCE_MINMAX_LINSPACE - num_threshs (int, optional): Number of thresholds between the min and max of the anomaly maps. - - Returns: - tuple[Tensor, Tensor]: - [0] Thresholds of shape (K,) and dtype is the same as `anomaly_maps.dtype`. - - [1] Binary classification matrices of shape (N, K, 2, 2) - - N: number of images/instances - K: number of thresholds - - The last two dimensions are the confusion matrix (ground truth, predictions) - So for each thresh it gives: - - `tp`: `[... , 1, 1]` - - `fp`: `[... , 0, 1]` - - `fn`: `[... , 1, 0]` - - `tn`: `[... , 0, 0]` - - `t` is for `true` and `f` is for `false`, `p` is for `positive` and `n` is for `negative`, so: - - `tp` stands for `true positive` - - `fp` stands for `false positive` - - `fn` stands for `false negative` - - `tn` stands for `true negative` - - The numbers in each confusion matrix are the counts of pixels in the image (not the ratios). - - Thresholds are shared across all images, so all confusion matrices, for instance, - at position [:, 0, :, :] are relative to the 1st threshold in `threshs`. - - Thresholds are sorted in ascending order. - """ - _validate.is_tensor(anomaly_maps, argname="anomaly_maps") - anomaly_maps_array = anomaly_maps.detach().cpu().numpy() - - _validate.is_tensor(masks, argname="masks") - masks_array = masks.detach().cpu().numpy() - - if threshs_given is not None: - _validate.is_tensor(threshs_given, argname="threshs_given") - threshs_given_array = threshs_given.detach().cpu().numpy() - else: - threshs_given_array = None - - threshs_array, binclf_curves_array = binclf_curve_numpy.per_image_binclf_curve( - anomaly_maps=anomaly_maps_array, - masks=masks_array, - threshs_choice=threshs_choice, - threshs_given=threshs_given_array, - num_threshs=num_threshs, - ) - threshs = torch.from_numpy(threshs_array).to(anomaly_maps.device) - binclf_curves = torch.from_numpy(binclf_curves_array).to(anomaly_maps.device).long() - - return threshs, binclf_curves - - -# =========================================== RATE METRICS =========================================== - - -def per_image_tpr(binclf_curves: Tensor) -> Tensor: - """Compute the true positive rates (TPR) for each image in the batch. - - Args: - binclf_curves (Tensor): Binary classification matrix curves (N, K, 2, 2). See `per_image_binclf_curve`. - - Returns: - Tensor: True positive rates (TPR) of shape (N, K) - - N: number of images/instances - K: number of thresholds - - The last dimension is the TPR for each threshold. - - Thresholds are sorted in ascending order, so TPR is in descending order. - """ - _validate_is_binclf_curves(binclf_curves) - binclf_curves_array = binclf_curves.detach().cpu().numpy() - tprs_array = binclf_curve_numpy.per_image_tpr(binclf_curves_array) - return torch.from_numpy(tprs_array).to(binclf_curves.device) - - -def per_image_fpr(binclf_curves: Tensor) -> Tensor: - """Compute the false positive rates (FPR) for each image in the batch. - - Args: - binclf_curves (Tensor): Binary classification matrix curves (N, K, 2, 2). See `per_image_binclf_curve`. - - Returns: - Tensor: False positive rates (FPR) of shape (N, K) - - N: number of images/instances - K: number of thresholds - - The last dimension is the FPR for each threshold. - - Thresholds are sorted in ascending order, so FPR is in descending order. - """ - _validate_is_binclf_curves(binclf_curves) - binclf_curves_array = binclf_curves.detach().cpu().numpy() - fprs_array = binclf_curve_numpy.per_image_fpr(binclf_curves_array) - return torch.from_numpy(fprs_array).to(binclf_curves.device) diff --git a/src/anomalib/metrics/per_image/pimo.py b/src/anomalib/metrics/per_image/pimo.py index 6c329f211a..bda8d800b6 100644 --- a/src/anomalib/metrics/per_image/pimo.py +++ b/src/anomalib/metrics/per_image/pimo.py @@ -39,26 +39,20 @@ # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -import json import logging from collections.abc import Sequence from dataclasses import dataclass, field -from pathlib import Path import torch from torch import Tensor from torchmetrics import Metric -from anomalib.data.utils.image import duplicate_filename from anomalib.data.utils.path import validate_path -from . import _validate, pimo_numpy, utils -from .utils import StatsOutliersPolicy, StatsRepeatedPolicy +from . import _validate, pimo_numpy logger = logging.getLogger(__name__) -# =========================================== AUX =========================================== - def _images_classes_from_masks(masks: Tensor) -> Tensor: masks = torch.concat(masks, dim=0) @@ -256,60 +250,6 @@ def thresh_at(self, fpr_level: float) -> tuple[int, float, float]: fpr_level, ) - def to_dict(self) -> dict[str, Tensor | str]: - """Return a dictionary with the result object's attributes.""" - dic = { - "threshs": self.threshs, - "shared_fpr": self.shared_fpr, - "per_image_tprs": self.per_image_tprs, - } - if self.paths is not None: - dic["paths"] = self.paths - return dic - - @classmethod - def from_dict(cls: type["PIMOResult"], dic: dict[str, Tensor | str | list[str]]) -> "PIMOResult": - """Return a result object from a dictionary.""" - try: - return cls(**dic) # type: ignore[arg-type] - - except TypeError as ex: - msg = f"Invalid input dictionary for {cls.__name__} object. Cause: {ex}." - raise TypeError(msg) from ex - - def save(self, file_path: str | Path) -> None: - """Save to a `.pt` file. - - Args: - file_path: path to the `.pt` file where to save the PIMO result. - If the file already exists, a numerical suffix is added to the filename. - """ - validate_path(file_path, should_exist=False, accepted_extensions=(".pt",)) - file_path = duplicate_filename(file_path) - payload = self.to_dict() - torch.save(payload, file_path) - - @classmethod - def load(cls: type["PIMOResult"], file_path: str | Path) -> "PIMOResult": - """Load from a `.pt` file. - - Args: - file_path: path to the `.pt` file where to load the PIMO result. - """ - validate_path(file_path, accepted_extensions=(".pt",)) - payload = torch.load(file_path) - if not isinstance(payload, dict): - msg = f"Invalid content in file {file_path}. Must be a dictionary." - raise TypeError(msg) - # for compatibility with the original code - if "shared_fpr_metric" in payload: - del payload["shared_fpr_metric"] - try: - return cls.from_dict(payload) - except TypeError as ex: - msg = f"Invalid content in file {file_path}. Cause: {ex}." - raise TypeError(msg) from ex - @dataclass class AUPIMOResult: @@ -448,97 +388,8 @@ def from_pimoresult( paths=paths, ) - def to_dict(self) -> dict[str, Tensor | str | float | int]: - """Return a dictionary with the result object's attributes.""" - dic = { - "fpr_lower_bound": self.fpr_lower_bound, - "fpr_upper_bound": self.fpr_upper_bound, - "num_threshs": self.num_threshs, - "thresh_lower_bound": self.thresh_lower_bound, - "thresh_upper_bound": self.thresh_upper_bound, - "aupimos": self.aupimos, - } - if self.paths is not None: - dic["paths"] = self.paths - return dic - - @classmethod - def from_dict(cls: type["AUPIMOResult"], dic: dict[str, Tensor | str | float | int | list[str]]) -> "AUPIMOResult": - """Return a result object from a dictionary.""" - try: - return cls(**dic) # type: ignore[arg-type] - - except TypeError as ex: - msg = f"Invalid input dictionary for {cls.__name__} object. Cause: {ex}." - raise TypeError(msg) from ex - - def save(self, file_path: str | Path) -> None: - """Save to a `.json` file. - - Args: - file_path: path to the `.json` file where to save the AUPIMO result. - If the file already exists, a numerical suffix is added to the filename. - """ - validate_path(file_path, should_exist=False, accepted_extensions=(".json",)) - file_path = duplicate_filename(file_path) - file_path = Path(file_path) - payload = self.to_dict() - aupimos: Tensor = payload["aupimos"] - payload["aupimos"] = aupimos.numpy().tolist() - with file_path.open("w") as f: - json.dump(payload, f, indent=4) - - @classmethod - def load(cls: type["AUPIMOResult"], file_path: str | Path) -> "AUPIMOResult": - """Load from a `.json` file. - - Args: - file_path: path to the `.json` file where to load the AUPIMO result. - """ - validate_path(file_path, accepted_extensions=(".json",)) - file_path = Path(file_path) - with file_path.open("r") as f: - payload = json.load(f) - if not isinstance(payload, dict): - file_path = str(file_path) - msg = f"Invalid payload in file {file_path}. Must be a dictionary." - raise TypeError(msg) - payload["aupimos"] = torch.tensor(payload["aupimos"], dtype=torch.float64) - # for compatibility with the original code - if "shared_fpr_metric" in payload: - del payload["shared_fpr_metric"] - try: - return cls.from_dict(payload) - except (TypeError, ValueError) as ex: - msg = f"Invalid payload in file {file_path}. Cause: {ex}." - raise TypeError(msg) from ex - - def stats( - self, - outliers_policy: str | StatsOutliersPolicy = StatsOutliersPolicy.NONE.value, - repeated_policy: str | StatsRepeatedPolicy = StatsRepeatedPolicy.AVOID.value, - repeated_replacement_atol: float = 1e-2, - ) -> list[dict[str, str | int | float]]: - """Return the AUPIMO statistics. - - See `anomalib.metrics.per_image.utils.per_image_scores_stats` for details. - - Returns: - list[dict[str, str | int | float]]: AUPIMO statistics - """ - return utils.per_image_scores_stats( - self.aupimos, - self.image_classes, - only_class=1, - outliers_policy=outliers_policy, - repeated_policy=repeated_policy, - repeated_replacement_atol=repeated_replacement_atol, - ) - # =========================================== FUNCTIONAL =========================================== - - def pimo_curves( anomaly_maps: Tensor, masks: Tensor, @@ -857,23 +708,6 @@ def normalizing_factor(fpr_bounds: tuple[float, float]) -> float: """ return pimo_numpy.aupimo_normalizing_factor(fpr_bounds) - @staticmethod - def random_model_score(fpr_bounds: tuple[float, float]) -> float: - """AUPIMO of a theoretical random model. - - "Random model" means that there is no discrimination between normal and anomalous pixels/patches/images. - It corresponds to assuming the functions T = F. - - For the FPR bounds (1e-5, 1e-4), the random model AUPIMO is ~4e-5. - - Args: - fpr_bounds: lower and upper bounds of the FPR integration range. - - Returns: - float: the AUPIMO score. - """ - return pimo_numpy.aupimo_random_model_score(fpr_bounds) - def __repr__(self) -> str: """Show the metric name and its integration bounds.""" lower, upper = self.fpr_bounds diff --git a/src/anomalib/metrics/per_image/pimo_numpy.py b/src/anomalib/metrics/per_image/pimo_numpy.py index d979cf1a53..2bd5c0cb89 100644 --- a/src/anomalib/metrics/per_image/pimo_numpy.py +++ b/src/anomalib/metrics/per_image/pimo_numpy.py @@ -11,7 +11,6 @@ # SPDX-License-Identifier: Apache-2.0 import logging -from enum import Enum import numpy as np from numpy import ndarray @@ -21,14 +20,6 @@ logger = logging.getLogger(__name__) -# =========================================== CONSTANTS =========================================== - - -class PIMOSharedFPRMetric(Enum): - """Shared FPR metric (x-axis of the PIMO curve).""" - - MEAN_PERIMAGE_FPR: str = "mean-per-image-fpr" - # =========================================== AUX =========================================== @@ -381,23 +372,3 @@ def aupimo_normalizing_factor(fpr_bounds: tuple[float, float]) -> float: fpr_lower_bound, fpr_upper_bound = fpr_bounds # the log's base must be the same as the one used in the integration! return float(np.log(fpr_upper_bound / fpr_lower_bound)) - - -def aupimo_random_model_score(fpr_bounds: tuple[float, float]) -> float: - """AUPIMO of a theoretical random model. - - "Random model" means that there is no discrimination between normal and anomalous pixels/patches/images. - It corresponds to assuming the functions T = F. - - For the FPR bounds (1e-5, 1e-4), the random model AUPIMO is ~4e-5. - - Args: - fpr_bounds: lower and upper bounds of the FPR integration range. - - Returns: - float: the AUPIMO score. - """ - _validate.is_rate_range(fpr_bounds) - fpr_lower_bound, fpr_upper_bound = fpr_bounds - integral_value = fpr_upper_bound - fpr_lower_bound - return float(integral_value / aupimo_normalizing_factor(fpr_bounds)) diff --git a/src/anomalib/metrics/per_image/utils.py b/src/anomalib/metrics/per_image/utils.py index 1d47674e2c..927ae4989f 100644 --- a/src/anomalib/metrics/per_image/utils.py +++ b/src/anomalib/metrics/per_image/utils.py @@ -245,8 +245,8 @@ def per_image_scores_stats( Outliers are handled according to `outliers_policy`: - None | "none": do not include outliers. - - "hi": only include high outliers. - - "lo": only include low outliers. + - "high": only include high outliers. + - "low": only include low outliers. - "both": include both high and low outliers. ** IMAGE INDEX ** diff --git a/src/anomalib/metrics/per_image/utils_numpy.py b/src/anomalib/metrics/per_image/utils_numpy.py index 7eb5413346..619e7c1677 100644 --- a/src/anomalib/metrics/per_image/utils_numpy.py +++ b/src/anomalib/metrics/per_image/utils_numpy.py @@ -29,14 +29,14 @@ class StatsOutliersPolicy(Enum): from the Q1 and Q3 quartiles (respectively low and high outliers). The IQR is the difference between Q3 and Q1. None | "none": do not include outliers. - "hi": only include high outliers. - "lo": only include low outliers. + "high": only include high outliers. + "low": only include low outliers. "both": include both high and low outliers. """ NONE: str = "none" - HI: str = "hi" - LO: str = "lo" + HIGH: str = "high" + LOW: str = "low" BOTH: str = "both" @@ -158,8 +158,8 @@ def per_image_scores_stats( Outliers are handled according to `outliers_policy`: - None | "none": do not include outliers. - - "hi": only include high outliers. - - "lo": only include low outliers. + - "high": only include high outliers. + - "low": only include low outliers. - "both": include both high and low outliers. ** IMAGE INDEX ** @@ -244,13 +244,13 @@ def per_image_scores_stats( outliers_lo = outliers[outliers < boxplot_stats["med"]] outliers_hi = outliers[outliers > boxplot_stats["med"]] - if outliers_policy in {StatsOutliersPolicy.HI, StatsOutliersPolicy.BOTH}: + if outliers_policy in {StatsOutliersPolicy.HIGH, StatsOutliersPolicy.BOTH}: boxplot_stats = { **boxplot_stats, **{f"outhi_{idx:06}": value for idx, value in enumerate(outliers_hi)}, } - if outliers_policy in {StatsOutliersPolicy.LO, StatsOutliersPolicy.BOTH}: + if outliers_policy in {StatsOutliersPolicy.LOW, StatsOutliersPolicy.BOTH}: boxplot_stats = { **boxplot_stats, **{f"outlo_{idx:06}": value for idx, value in enumerate(outliers_lo)}, diff --git a/tests/unit/metrics/per_image/test_binclf_curve.py b/tests/unit/metrics/per_image/test_binclf_curve.py index 62112bc257..cd7c0cdd98 100644 --- a/tests/unit/metrics/per_image/test_binclf_curve.py +++ b/tests/unit/metrics/per_image/test_binclf_curve.py @@ -11,11 +11,9 @@ import numpy as np import pytest -import torch from numpy import ndarray -from torch import Tensor -from anomalib.metrics.per_image import binclf_curve, binclf_curve_numpy +from anomalib.metrics.per_image import binclf_curve_numpy def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: @@ -233,24 +231,6 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: argvalues=per_image_binclf_curves_numpy_argvalues, ) - # the test with the torch interface are the same we just convert ndarray to Tensor - if metafunc.function is test_per_image_binclf_curve_torch: - metafunc.parametrize( - argnames=( - "anomaly_maps", - "masks", - "threshs_choice", - "threshs_given", - "num_threshs", - "expected_threshs", - "expected_binclf_curves", - ), - argvalues=[ - tuple(torch.from_numpy(v) if isinstance(v, np.ndarray) else v for v in argvals) - for argvals in per_image_binclf_curves_numpy_argvalues - ], - ) - if metafunc.function is test_per_image_binclf_curve_numpy_validations: metafunc.parametrize( argnames=("args", "exception"), @@ -305,14 +285,6 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: ], ) - if metafunc.function is test_rate_metrics_torch: - metafunc.parametrize( - argnames=("binclf_curves", "expected_fprs", "expected_tprs"), - argvalues=[ - (torch.from_numpy(binclf_curves), torch.from_numpy(expected_fprs), torch.from_numpy(expected_tprs)), - ], - ) - # ================================================================================================== # LOW-LEVEL FUNCTIONS (PYTHON) @@ -429,48 +401,3 @@ def test_rate_metrics_numpy(binclf_curves: ndarray, expected_fprs: ndarray, expe assert np.allclose(tprs, expected_tprs, equal_nan=True) assert np.allclose(fprs, expected_fprs, equal_nan=True) - - -# ================================================================================================== -# API FUNCTIONS (TORCH) - - -def test_per_image_binclf_curve_torch( - anomaly_maps: Tensor, - masks: Tensor, - threshs_choice: str, - threshs_given: Tensor | None, - num_threshs: int | None, - expected_threshs: Tensor, - expected_binclf_curves: Tensor, -) -> None: - """Test if `per_image_binclf_curve()` returns the expected values.""" - computed_threshs, computed_binclf_curves = binclf_curve.per_image_binclf_curve( - anomaly_maps, - masks, - threshs_choice=threshs_choice, - threshs_given=threshs_given, - num_threshs=num_threshs, - ) - - # threshs - assert computed_threshs.shape == expected_threshs.shape - assert computed_threshs.dtype == computed_threshs.dtype - assert (computed_threshs == expected_threshs).all() - - # binclf_curves - assert computed_binclf_curves.shape == expected_binclf_curves.shape - assert computed_binclf_curves.dtype == expected_binclf_curves.dtype - assert (computed_binclf_curves == expected_binclf_curves).all() - - -def test_rate_metrics_torch(binclf_curves: Tensor, expected_fprs: Tensor, expected_tprs: Tensor) -> None: - """Test if rate metrics are computed correctly.""" - tprs = binclf_curve.per_image_tpr(binclf_curves) - fprs = binclf_curve.per_image_fpr(binclf_curves) - - assert tprs.shape == expected_tprs.shape - assert fprs.shape == expected_fprs.shape - - assert torch.allclose(tprs, expected_tprs, equal_nan=True) - assert torch.allclose(fprs, expected_fprs, equal_nan=True) diff --git a/tests/unit/metrics/per_image/test_pimo.py b/tests/unit/metrics/per_image/test_pimo.py index 9e818346fb..d0092fb616 100644 --- a/tests/unit/metrics/per_image/test_pimo.py +++ b/tests/unit/metrics/per_image/test_pimo.py @@ -7,9 +7,6 @@ # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -import tempfile -from pathlib import Path - import numpy as np import pytest import torch @@ -19,8 +16,6 @@ from anomalib.metrics.per_image import pimo, pimo_numpy from anomalib.metrics.per_image.pimo import AUPIMOResult, PIMOResult -from .test_utils import assert_statsdict_stuff - def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: """Generate tests for all functions in this module. @@ -186,12 +181,6 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: ], ) - if metafunc.function is test_pimoresult_object or metafunc.function is test_aupimoresult_object: - anomaly_maps = torch.from_numpy(anomaly_maps) - masks = torch.from_numpy(masks) - metafunc.parametrize(argnames=("anomaly_maps", "masks"), argvalues=[(anomaly_maps, masks)]) - metafunc.parametrize(argnames=("paths",), argvalues=[(None,), (["/path/to/a", "/path/to/b", "/path/to/c"],)]) - def _do_test_pimo_outputs( threshs: ndarray | Tensor, @@ -488,104 +477,3 @@ def test_aupimo_edge( force=False, **fpr_bounds, ) - - -def test_pimoresult_object( - anomaly_maps: Tensor, - masks: Tensor, - paths: list[str] | None, -) -> None: - """Test if `PIMOResult` can be converted to other formats and back.""" - optional_kwargs = {} - if paths is not None: - optional_kwargs["paths"] = paths - - pimoresult = pimo.pimo_curves( - anomaly_maps, - masks, - num_threshs=7, - **optional_kwargs, - ) - - _ = pimoresult.num_threshs - _ = pimoresult.num_images - _ = pimoresult.image_classes - - # object -> dict -> object - dic = pimoresult.to_dict() - assert isinstance(dic, dict) - pimoresult_from_dict = PIMOResult.from_dict(dic) - assert isinstance(pimoresult_from_dict, PIMOResult) - # values should be the same - assert torch.allclose(pimoresult_from_dict.threshs, pimoresult.threshs) - assert torch.allclose(pimoresult_from_dict.shared_fpr, pimoresult.shared_fpr) - assert torch.allclose(pimoresult_from_dict.per_image_tprs, pimoresult.per_image_tprs, equal_nan=True) - - # object -> file -> object - with tempfile.TemporaryDirectory() as tmpdir: - file_path = Path(tmpdir) / "pimo.pt" - pimoresult.save(str(file_path)) - assert file_path.exists() - pimoresult_from_load = PIMOResult.load(str(file_path)) - assert isinstance(pimoresult_from_load, PIMOResult) - # values should be the same - assert torch.allclose(pimoresult_from_load.threshs, pimoresult.threshs) - assert torch.allclose(pimoresult_from_load.shared_fpr, pimoresult.shared_fpr) - assert torch.allclose(pimoresult_from_load.per_image_tprs, pimoresult.per_image_tprs, equal_nan=True) - - -def test_aupimoresult_object( - anomaly_maps: Tensor, - masks: Tensor, - paths: list[str] | None, -) -> None: - """Test if `AUPIMOResult` can be converted to other formats and back.""" - optional_kwargs = {} - if paths is not None: - optional_kwargs["paths"] = paths - - _, aupimoresult = pimo.aupimo_scores( - anomaly_maps, - masks, - num_threshs=7, - fpr_bounds=(1e-5, 1e-4), - force=True, - **optional_kwargs, - ) - - # call properties - _ = aupimoresult.num_images - _ = aupimoresult.image_classes - _ = aupimoresult.fpr_bounds - _ = aupimoresult.thresh_bounds - - # object -> dict -> object - dic = aupimoresult.to_dict() - assert isinstance(dic, dict) - aupimoresult_from_dict = AUPIMOResult.from_dict(dic) - assert isinstance(aupimoresult_from_dict, AUPIMOResult) - # values should be the same - assert aupimoresult_from_dict.fpr_bounds == aupimoresult.fpr_bounds - assert aupimoresult_from_dict.num_threshs == aupimoresult.num_threshs - assert aupimoresult_from_dict.thresh_bounds == aupimoresult.thresh_bounds - assert torch.allclose(aupimoresult_from_dict.aupimos, aupimoresult.aupimos, equal_nan=True) - - # object -> file -> object - with tempfile.TemporaryDirectory() as tmpdir: - file_path = Path(tmpdir) / "aupimo.json" - aupimoresult.save(str(file_path)) - assert file_path.exists() - aupimoresult_from_load = AUPIMOResult.load(str(file_path)) - assert isinstance(aupimoresult_from_load, AUPIMOResult) - # values should be the same - assert aupimoresult_from_load.fpr_bounds == aupimoresult.fpr_bounds - assert aupimoresult_from_load.num_threshs == aupimoresult.num_threshs - assert aupimoresult_from_load.thresh_bounds == aupimoresult.thresh_bounds - assert torch.allclose(aupimoresult_from_load.aupimos, aupimoresult.aupimos, equal_nan=True) - - # statistics - stats = aupimoresult.stats() - assert len(stats) == 6 - - for statdic in stats: - assert_statsdict_stuff(statdic, 2) diff --git a/tests/unit/metrics/per_image/test_utils.py b/tests/unit/metrics/per_image/test_utils.py index 0c894bf56f..c8d5943921 100644 --- a/tests/unit/metrics/per_image/test_utils.py +++ b/tests/unit/metrics/per_image/test_utils.py @@ -118,9 +118,9 @@ def test_per_image_scores_stats() -> None: stats = per_image_scores_stats(scores, outliers_policy=StatsOutliersPolicy.BOTH) assert len(stats) == 6 - stats = per_image_scores_stats(scores, outliers_policy=StatsOutliersPolicy.LO) + stats = per_image_scores_stats(scores, outliers_policy=StatsOutliersPolicy.LOW) assert len(stats) == 6 - stats = per_image_scores_stats(scores, outliers_policy=StatsOutliersPolicy.HI) + stats = per_image_scores_stats(scores, outliers_policy=StatsOutliersPolicy.HIGH) assert len(stats) == 6 stats = per_image_scores_stats(scores, outliers_policy=StatsOutliersPolicy.NONE) assert len(stats) == 6