Additional metrics during train (#194)

* Added additional metrics to fit dictionary

* Added in test also

Fix mypy and flake after rebase, added random state to mixup and cutout and changs no resampling for new code

fix bug in setup.py

Author: ravinkohli

autoPyTorch/datasets/base_dataset.py

@@ -22,9 +22,9 @@
     HoldOutFunc,
     HoldOutFuncs,
     HoldoutValTypes,
-    get_no_resampling_validators,
-    NoResamplingStrategyTypes,
-    NO_RESAMPLING_FN
+    NoResamplingFunc,
+    NoResamplingFuncs,
+    NoResamplingStrategyTypes
 )
 from autoPyTorch.utils.common import FitRequirement
 
@@ -114,24 +114,19 @@ def __init__(
             val_transforms (Optional[torchvision.transforms.Compose]):
                 Additional Transforms to be applied to the validation/test data
         """
-        self.dataset_name = dataset_name
 
-        if self.dataset_name is None:
+        if dataset_name is None:
             self.dataset_name = str(uuid.uuid1(clock_seq=os.getpid()))
+        else:
+            self.dataset_name = dataset_name
 
         if not hasattr(train_tensors[0], 'shape'):
             type_check(train_tensors, val_tensors)
         self.train_tensors, self.val_tensors, self.test_tensors = train_tensors, val_tensors, test_tensors
-<<<<<<< HEAD
         self.cross_validators: Dict[str, CrossValFunc] = {}
         self.holdout_validators: Dict[str, HoldOutFunc] = {}
         self.random_state = np.random.RandomState(seed=seed)
-=======
-        self.cross_validators: Dict[str, CROSS_VAL_FN] = {}
-        self.holdout_validators: Dict[str, HOLDOUT_FN] = {}
-        self.no_resampling_validators: Dict[str, NO_RESAMPLING_FN] = {}
-        self.rng = np.random.RandomState(seed=seed)
->>>>>>> Fix mypy and flake
+        self.no_resampling_validators: Dict[str, NoResamplingFunc] = {}
         self.shuffle = shuffle
         self.resampling_strategy = resampling_strategy
         self.resampling_strategy_args = resampling_strategy_args
@@ -156,7 +151,7 @@ def __init__(
         # Make sure cross validation splits are created once
         self.cross_validators = CrossValFuncs.get_cross_validators(*CrossValTypes)
         self.holdout_validators = HoldOutFuncs.get_holdout_validators(*HoldoutValTypes)
-        self.no_resampling_validators = get_no_resampling_validators(*NoResamplingStrategyTypes)
+        self.no_resampling_validators = NoResamplingFuncs.get_no_resampling_validators(*NoResamplingStrategyTypes)
 
         self.splits = self.get_splits_from_resampling_strategy()
 
@@ -257,7 +252,8 @@ def get_splits_from_resampling_strategy(self) -> List[Tuple[List[int], Optional[
                 )
             )
         elif isinstance(self.resampling_strategy, NoResamplingStrategyTypes):
-            splits.append((self.no_resampling_validators[self.resampling_strategy.name](self._get_indices()), None))
+            splits.append((self.no_resampling_validators[self.resampling_strategy.name](self.random_state,
+                                                                                        self._get_indices()), None))
         else:
             raise ValueError(f"Unsupported resampling strategy={self.resampling_strategy}")
         return splits

autoPyTorch/datasets/resampling_strategy.py

@@ -32,8 +32,10 @@ def __call__(self, random_state: np.random.RandomState, val_share: float,
         ...
 
 
-class NO_RESAMPLING_FN(Protocol):
-    def __call__(self, indices: np.ndarray) -> np.ndarray:
+class NoResamplingFunc(Protocol):
+    def __call__(self,
+                 random_state: np.random.RandomState,
+                 indices: np.ndarray) -> np.ndarray:
         ...
 
 
@@ -244,53 +246,46 @@ def get_cross_validators(cls, *cross_val_types: CrossValTypes) -> Dict[str, Cros
         return cross_validators
 
 
-def get_no_resampling_validators(*no_resampling: NoResamplingStrategyTypes) -> Dict[str, NO_RESAMPLING_FN]:
-    no_resampling_strategies = {}  # type: Dict[str, NO_RESAMPLING_FN]
-    for strategy in no_resampling:
-        no_resampling_fn = globals()[strategy.name]
-        no_resampling_strategies[strategy.name] = no_resampling_fn
-    return no_resampling_strategies
+class NoResamplingFuncs():
+    @classmethod
+    def get_no_resampling_validators(cls, *no_resampling_types: NoResamplingStrategyTypes
+                                     ) -> Dict[str, NoResamplingFunc]:
+        no_resampling_strategies: Dict[str, NoResamplingFunc] = {
+            no_resampling_type.name: getattr(cls, no_resampling_type.name)
+            for no_resampling_type in no_resampling_types
+        }
+        return no_resampling_strategies
 
+    @staticmethod
+    def no_resampling(random_state: np.random.RandomState,
+                      indices: np.ndarray) -> np.ndarray:
+        """
+        Returns the indices without performing
+        any operation on them. To be used for
+        fitting on the whole dataset.
+        This strategy is not compatible with
+        HPO search.
+        Args:
+            indices:  array of indices
 
-def no_resampling(indices: np.ndarray) -> np.ndarray:
-    """
-    Returns the indices without performing
-    any operation on them. To be used for
-    fitting on the whole dataset.
-    This strategy is not compatible with
-    HPO search.
-    Args:
-        indices:  array of indices
-
-    Returns:
-        np.ndarray: array of indices
-    """
-    return indices
+        Returns:
+            np.ndarray: array of indices
+        """
+        return indices
 
+    @staticmethod
+    def shuffle_no_resampling(random_state: np.random.RandomState,
+                              indices: np.ndarray) -> np.ndarray:
+        """
+        Returns the indices after shuffling them.
+        To be used for fitting on the whole dataset.
+        This strategy is not compatible with HPO search.
+        Args:
+            random_state: random state
+            indices:  array of indices
 
-def shuffle_no_resampling(indices: np.ndarray, **kwargs: Any) -> np.ndarray:
-    """
-    Returns the indices after shuffling them.
-    To be used for fitting on the whole dataset.
-    This strategy is not compatible with HPO search.
-    Args:
-        indices:  array of indices
-
-    Returns:
-        np.ndarray: shuffled array of indices
-    """
-    if 'random_state' in kwargs:
-        if isinstance(kwargs['random_state'], np.random.RandomState):
-            kwargs['random_state'].shuffle(indices)
-        elif isinstance(kwargs['random_state'], int):
-            np.random.seed(kwargs['random_state'])
-            np.random.shuffle(indices)
-        else:
-            raise ValueError("Illegal value for 'random_state' entered. "
-                             "Expected it to be {} or {} but got {}".format(int,
-                                                                            np.random.RandomState,
-                                                                            type(kwargs['random_state'])))
-    else:
-        np.random.shuffle(indices)
-
-    return indices
+        Returns:
+            np.ndarray: shuffled array of indices
+        """
+        random_state.shuffle(indices)
+        return indices

autoPyTorch/pipeline/components/training/trainer/cutout_utils.py

@@ -10,6 +10,8 @@
 
 import numpy as np
 
+from sklearn.utils import check_random_state
+
 from autoPyTorch.constants import CLASSIFICATION_TASKS, STRING_TO_TASK_TYPES
 from autoPyTorch.pipeline.components.training.trainer.utils import Lookahead
 from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter
@@ -35,7 +37,12 @@ def __init__(self, patch_ratio: float,
         """
         self.use_stochastic_weight_averaging = use_stochastic_weight_averaging
         self.weighted_loss = weighted_loss
-        self.random_state = random_state
+        if random_state is None:
+            # A trainer components need a random state for
+            # sampling -- for example in MixUp training
+            self.random_state = check_random_state(1)
+        else:
+            self.random_state = random_state
         self.use_snapshot_ensemble = use_snapshot_ensemble
         self.se_lastk = se_lastk
         self.use_lookahead_optimizer = use_lookahead_optimizer

autoPyTorch/pipeline/components/training/trainer/mixup_utils.py

@@ -10,6 +10,8 @@
 
 import numpy as np
 
+from sklearn.utils import check_random_state
+
 from autoPyTorch.constants import CLASSIFICATION_TASKS, STRING_TO_TASK_TYPES
 from autoPyTorch.pipeline.components.training.trainer.utils import Lookahead
 from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter
@@ -34,7 +36,12 @@ def __init__(self, alpha: float,
         """
         self.use_stochastic_weight_averaging = use_stochastic_weight_averaging
         self.weighted_loss = weighted_loss
-        self.random_state = random_state
+        if random_state is None:
+            # A trainer components need a random state for
+            # sampling -- for example in MixUp training
+            self.random_state = check_random_state(1)
+        else:
+            self.random_state = random_state
         self.use_snapshot_ensemble = use_snapshot_ensemble
         self.se_lastk = se_lastk
         self.use_lookahead_optimizer = use_lookahead_optimizer

setup.py

@@ -58,7 +58,7 @@
             "pre-commit",
             "pytest-cov",
             'pytest-forked',
-            "pytest-mock"
+            "pytest-mock",
             "codecov",
             "pep8",
             "mypy",

test/test_api/test_api.py

@@ -35,7 +35,7 @@
 from autoPyTorch.pipeline.components.setup.traditional_ml.traditional_learner import _traditional_learners
 from autoPyTorch.pipeline.components.training.metrics.metrics import accuracy
 
-from test.test_api.api_utils import print_debug_information
+from test.test_api.api_utils import print_debug_information  # noqa E402
 
 
 CV_NUM_SPLITS = 2