[FIX] ROC AUC for multi class classification

autoPyTorch/api/base_task.py

@@ -978,7 +978,7 @@ def _search(
         smac_scenario_args: Optional[Dict[str, Any]] = None,
         get_smac_object_callback: Optional[Callable] = None,
         tae_func: Optional[Callable] = None,
-        all_supported_metrics: bool = True,
+        all_supported_metrics: bool = False,
         precision: int = 32,
         disable_file_output: Optional[List[Union[str, DisableFileOutputParameters]]] = None,
         load_models: bool = True,
@@ -1076,7 +1076,7 @@ def _search(
                 TargetAlgorithm to be optimised. If None, `eval_function`
                 available in autoPyTorch/evaluation/train_evaluator is used.
                 Must be child class of AbstractEvaluator.
-            all_supported_metrics (bool: default=True):
+            all_supported_metrics (bool: default=False):
                 If True, all metrics supporting current task will be calculated
                 for each pipeline and results will be available via cv_results
             precision (int: default=32):

autoPyTorch/api/tabular_classification.py

@@ -254,7 +254,7 @@ def search(
         memory_limit: int = 4096,
         smac_scenario_args: Optional[Dict[str, Any]] = None,
         get_smac_object_callback: Optional[Callable] = None,
-        all_supported_metrics: bool = True,
+        all_supported_metrics: bool = False,
         precision: int = 32,
         disable_file_output: Optional[List[Union[str, DisableFileOutputParameters]]] = None,
         load_models: bool = True,
@@ -354,7 +354,7 @@ def search(
                 TargetAlgorithm to be optimised. If None, `eval_function`
                 available in autoPyTorch/evaluation/train_evaluator is used.
                 Must be child class of AbstractEvaluator.
-            all_supported_metrics (bool: default=True):
+            all_supported_metrics (bool: default=False):
                 If True, all metrics supporting current task will be calculated
                 for each pipeline and results will be available via cv_results
             precision (int: default=32):

autoPyTorch/api/tabular_regression.py

@@ -253,7 +253,7 @@ def search(
         memory_limit: int = 4096,
         smac_scenario_args: Optional[Dict[str, Any]] = None,
         get_smac_object_callback: Optional[Callable] = None,
-        all_supported_metrics: bool = True,
+        all_supported_metrics: bool = False,
         precision: int = 32,
         disable_file_output: Optional[List[Union[str, DisableFileOutputParameters]]] = None,
         load_models: bool = True,
@@ -353,7 +353,7 @@ def search(
                 TargetAlgorithm to be optimised. If None, `eval_function`
                 available in autoPyTorch/evaluation/train_evaluator is used.
                 Must be child class of AbstractEvaluator.
-            all_supported_metrics (bool: default=True):
+            all_supported_metrics (bool: default=False):
                 If True, all metrics supporting current task will be calculated
                 for each pipeline and results will be available via cv_results
             precision (int: default=32):

autoPyTorch/api/time_series_forecasting.py

@@ -289,7 +289,7 @@ def search(
         memory_limit: Optional[int] = 4096,
         smac_scenario_args: Optional[Dict[str, Any]] = None,
         get_smac_object_callback: Optional[Callable] = None,
-        all_supported_metrics: bool = True,
+        all_supported_metrics: bool = False,
         precision: int = 32,
         disable_file_output: List = [],
         load_models: bool = True,
@@ -396,7 +396,7 @@ def search(
                 instances, num_params, runhistory, seed and ta. This is
                 an advanced feature. Use only if you are familiar with
                 [SMAC](https://automl.github.io/SMAC3/master/index.html).
-            all_supported_metrics (bool), (default=True): if True, all
+            all_supported_metrics (bool), (default=False): if True, all
                 metrics supporting current task will be calculated
                 for each pipeline and results will be available via cv_results
             precision (int), (default=32): Numeric precision used when loading

autoPyTorch/pipeline/components/setup/network/base_network.py

@@ -56,15 +56,14 @@ def fit(self, X: Dict[str, Any], y: Any = None) -> autoPyTorchTrainingComponent:
 
         self.network = torch.nn.Sequential(X['network_embedding'], X['network_backbone'], X['network_head'])
 
+        if STRING_TO_TASK_TYPES[X['dataset_properties']['task_type']] in CLASSIFICATION_TASKS:
+            self.network = torch.nn.Sequential(self.network, nn.Softmax(dim=1))
         # Properly set the network training device
         if self.device is None:
             self.device = get_device_from_fit_dictionary(X)
 
         self.to(self.device)
 
-        if STRING_TO_TASK_TYPES[X['dataset_properties']['task_type']] in CLASSIFICATION_TASKS:
-            self.final_activation = nn.Softmax(dim=1)
-
         self.is_fitted_ = True
 
         return self

autoPyTorch/pipeline/components/training/metrics/base.py

@@ -173,7 +173,7 @@ def __call__(
                 Score function applied to prediction of estimator on X.
         """
         y_type = type_of_target(y_true)
-        if y_type not in ("binary", "multilabel-indicator"):
+        if y_type not in ("binary", "multilabel-indicator") and self.name != 'roc_auc':
             raise ValueError("{0} format is not supported".format(y_type))
 
         if y_type == "binary":

autoPyTorch/pipeline/components/training/metrics/metrics.py

@@ -57,7 +57,7 @@
 
 
 # Score functions that need decision values
-roc_auc = make_metric('roc_auc', sklearn.metrics.roc_auc_score, needs_threshold=True)
+roc_auc = make_metric('roc_auc', sklearn.metrics.roc_auc_score, needs_threshold=True, multi_class= 'ovo')
 average_precision = make_metric('average_precision',
                                 sklearn.metrics.average_precision_score,
                                 needs_threshold=True)

autoPyTorch/pipeline/components/training/metrics/utils.py

@@ -99,8 +99,8 @@ def get_metrics(dataset_properties: Dict[str, Any],
     if names is not None:
         for name in names:
             if name not in supported_metrics.keys():
-                raise ValueError("Invalid name entered for task {}, currently "
-                                 "supported metrics for task include {}".format(dataset_properties['task_type'],
+                raise ValueError("Invalid name {} entered for task {}, currently "
+                                 "supported metrics for task include {}".format(name, dataset_properties['task_type'],
                                                                                 list(supported_metrics.keys())))
             else:
                 metric = supported_metrics[name]

autoPyTorch/pipeline/components/training/trainer/RowCutMixTrainer.py

@@ -37,17 +37,18 @@ def data_preparation(self, X: np.ndarray, y: np.ndarray,
         if beta <= 0 or r > self.alpha:
             return X, {'y_a': y, 'y_b': y[shuffled_indices], 'lam': 1}
 
-        cut_column_indices = torch.as_tensor(
-            self.random_state.choice(
-                range(n_columns),
-                max(1, np.int32(n_columns * lam)),
-                replace=False,
-            ),
-        )
 
         # Replace the values in `cut_indices` columns with
         # the values from `permed_indices`
-        X[:, cut_column_indices] = X[shuffled_indices, :][:, cut_column_indices]
+        for i, idx in enumerate(shuffled_indices):
+            cut_column_indices = torch.as_tensor(
+                self.random_state.choice(
+                    range(n_columns),
+                    max(1, np.int32(n_columns * lam)),
+                    replace=False,
+                ),
+            )
+            X[i, cut_column_indices] = X[idx, cut_column_indices]
 
         # Since we cannot cut exactly `lam x 100 %` of rows, we need to adjust the `lam`
         lam = 1 - (len(cut_column_indices) / n_columns)

autoPyTorch/pipeline/components/training/trainer/RowCutOutTrainer.py

@@ -39,15 +39,17 @@ def data_preparation(self, X: np.ndarray, y: np.ndarray,
             lam = 1
             return X, {'y_a': y_a, 'y_b': y_b, 'lam': lam}
 
-        size: int = np.shape(X)[1]
-        cut_column_indices = self.random_state.choice(
-            range(size),
-            max(1, np.int32(size * self.patch_ratio)),
-            replace=False,
-        )
+        n_rows, size = np.shape(X)
+        for i in range(n_rows):
+            cut_column_indices = self.random_state.choice(
+                range(size),
+                max(1, np.int32(size * self.patch_ratio)),
+                replace=False,
+            )
+            X[i, cut_column_indices] = 0
+
 
         # Mask the selected features as 0
-        X[:, cut_column_indices] = 0
         lam = 1
         y_a = y
         y_b = y