From e138b6973f6901584cfb42b056963fa982966f4b Mon Sep 17 00:00:00 2001 From: chico Date: Fri, 19 Feb 2021 02:02:43 +0100 Subject: [PATCH 01/50] First push for Mix/cut regularization Fix mypy Initial implementation of adversarial training Modifying the code to have activation controlled batch normalization Adding activation controlled weight decay, updating the style for code style check Commit for passing style check Style check try 2 Bug fix Adding unit test for adversarial trainer Adding code for activation controlled skip connections, with the additional choice of shake-shake and shake-drop being hyperparameters as the choice for the multi branch networks Bug fix for the failing tests Adding better conditions Try at a fix Temporary fix for the failing test Failing code check Failing code check v2 Add new update to fix break Flake8 coding style fix Removing duplicate unit test In progress --- .../setup/network_backbone/ResNetBackbone.py | 105 +++++-- .../network_backbone/ShapedResNetBackbone.py | 49 +-- .../setup/optimizer/AdamOptimizer.py | 27 +- .../setup/optimizer/AdamWOptimizer.py | 28 +- .../setup/optimizer/RMSpropOptimizer.py | 26 +- .../setup/optimizer/SGDOptimizer.py | 28 +- .../training/trainer/AdversarialTrainer.py | 158 ++++++++++ .../training/trainer/GridCutMixTrainer.py | 68 ++++ .../training/trainer/GridCutOutTrainer.py | 56 ++++ .../training/trainer/MixUpTrainer.py | 58 +--- .../training/trainer/RowCutMixTrainer.py | 63 ++++ .../training/trainer/RowCutOutTrainer.py | 63 ++++ .../training/trainer/StandardTrainer.py | 9 +- .../components/training/trainer/__init__.py | 62 ++++ .../training/trainer/base_trainer.py | 25 +- .../training/trainer/cutout_utils.py | 62 ++++ .../training/trainer/mixup_utils.py | 51 +++ .../components/training/test_training.py | 293 +++++++++++++----- 18 files changed, 1030 insertions(+), 201 deletions(-) create mode 100644 autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py create mode 100644 autoPyTorch/pipeline/components/training/trainer/GridCutMixTrainer.py create mode 100644 autoPyTorch/pipeline/components/training/trainer/GridCutOutTrainer.py create mode 100644 autoPyTorch/pipeline/components/training/trainer/RowCutMixTrainer.py create mode 100644 autoPyTorch/pipeline/components/training/trainer/RowCutOutTrainer.py create mode 100644 autoPyTorch/pipeline/components/training/trainer/cutout_utils.py create mode 100644 autoPyTorch/pipeline/components/training/trainer/mixup_utils.py diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py index 4dbc41618..8cb6d5260 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py @@ -45,8 +45,8 @@ def build_backbone(self, input_shape: Tuple[int, ...]) -> torch.nn.Sequential: dropout=self.config[f'dropout_{i}'] if self.config['use_dropout'] else None, ) ) - - layers.append(nn.BatchNorm1d(self.config["num_units_%i" % self.config['num_groups']])) + if self.config['use_batch_norm']: + layers.append(nn.BatchNorm1d(self.config["num_units_%i" % self.config['num_groups']])) layers.append(_activations[self.config["activation"]]()) backbone = nn.Sequential(*layers) return backbone @@ -104,6 +104,18 @@ def get_hyperparameter_search_space( value_range=(True, False), default_value=False, ), + use_batch_norm: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_batch_norm", + value_range=(True, False), + default_value=False, + ), + use_skip_connection: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_skip_connection", + value_range=(True, False), + default_value=True, + ), + multi_branch_choice: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="mb_choice", + value_range=('None', 'shake-shake', 'shake-drop'), + default_value='shake-drop', + ), num_units: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="num_units", value_range=(10, 1024), default_value=200, @@ -138,22 +150,28 @@ def get_hyperparameter_search_space( # The number of groups that will compose the resnet. That is, # a group can have N Resblock. The M number of this N resblock # repetitions is num_groups - min_num_gropus, max_num_groups = num_groups.value_range + _, max_num_groups = num_groups.value_range num_groups = get_hyperparameter(num_groups, UniformIntegerHyperparameter) add_hyperparameter(cs, activation, CategoricalHyperparameter) cs.add_hyperparameters([num_groups]) + # activation controlled batch normalization + add_hyperparameter(cs, use_batch_norm, CategoricalHyperparameter) + # We can have dropout in the network for # better generalization use_dropout = get_hyperparameter(use_dropout, CategoricalHyperparameter) cs.add_hyperparameters([use_dropout]) - - use_shake_shake = get_hyperparameter(use_shake_shake, CategoricalHyperparameter) - use_shake_drop = get_hyperparameter(use_shake_drop, CategoricalHyperparameter) + + use_sc = get_hyperparameter(use_skip_connection, CategoricalHyperparameter) + mb_choice = get_hyperparameter(multi_branch_choice, CategoricalHyperparameter) shake_drop_prob = get_hyperparameter(max_shake_drop_probability, UniformFloatHyperparameter) - cs.add_hyperparameters([use_shake_shake, use_shake_drop, shake_drop_prob]) - cs.add_condition(CS.EqualsCondition(shake_drop_prob, use_shake_drop, True)) + cs.add_hyperparameters([use_sc, mb_choice, shake_drop_prob]) + cs.add_condition(CS.EqualsCondition(mb_choice, use_sc, True)) + # TODO check if shake_drop is as an option in mb_choice + # Incomplete work + cs.add_condition(CS.EqualsCondition(shake_drop_prob, mb_choice, "shake-drop")) # It is the upper bound of the nr of groups, # since the configuration will actually be sampled. @@ -223,28 +241,38 @@ def __init__( # as well (start_norm) if in_features != out_features: self.shortcut = nn.Linear(in_features, out_features) - self.start_norm = nn.Sequential( - nn.BatchNorm1d(in_features), + initial_normalization = list() + if self.config['use_batch_norm']: + initial_normalization.append( + nn.BatchNorm1d(in_features) + ) + initial_normalization.append( self.activation() ) + self.start_norm = nn.Sequential( + *initial_normalization + ) self.block_index = block_index self.num_blocks = blocks_per_group * self.config["num_groups"] self.layers = self._build_block(in_features, out_features) - if config["use_shake_shake"]: - self.shake_shake_layers = self._build_block(in_features, out_features) + if self.config["use_skip_connection"]: + if config["multi_branch_choice"] == 'shake-shake': + self.shake_shake_layers = self._build_block(in_features, out_features) - # each bloack consists of two linear layers with batch norm and activation + # each block consists of two linear layers with batch norm and activation def _build_block(self, in_features: int, out_features: int) -> nn.Module: layers = list() if self.start_norm is None: - layers.append(nn.BatchNorm1d(in_features)) + if self.config['use_batch_norm']: + layers.append(nn.BatchNorm1d(in_features)) layers.append(self.activation()) layers.append(nn.Linear(in_features, out_features)) - layers.append(nn.BatchNorm1d(out_features)) + if self.config['use_batch_norm']: + layers.append(nn.BatchNorm1d(out_features)) layers.append(self.activation()) if self.config["use_dropout"]: @@ -254,7 +282,9 @@ def _build_block(self, in_features: int, out_features: int) -> nn.Module: return nn.Sequential(*layers) def forward(self, x: torch.FloatTensor) -> torch.FloatTensor: - residual = x + + if self.config["use_skip_connection"]: + residual = x # if shortcut is not none we need a layer such that x matches the output dimension if self.shortcut is not None and self.start_norm is not None: @@ -267,26 +297,33 @@ def forward(self, x: torch.FloatTensor) -> torch.FloatTensor: # if in_features != out_features # -> result = W_shortcut(A(BN(x))) + W_2(~D(A(BN(W_1(A(BN(x)))))) x = self.start_norm(x) - residual = self.shortcut(x) - - if self.config["use_shake_shake"]: - x1 = self.layers(x) - x2 = self.shake_shake_layers(x) - alpha, beta = shake_get_alpha_beta(self.training, x.is_cuda) - x = shake_shake(x1, x2, alpha, beta) + if self.config["use_skip_connection"]: + residual = self.shortcut(x) + + if self.config["use_skip_connection"]: + if self.config["multi_branch_choice"] == 'shake-shake': + x1 = self.layers(x) + x2 = self.shake_shake_layers(x) + alpha, beta = shake_get_alpha_beta(self.training, x.is_cuda) + x = shake_shake(x1, x2, alpha, beta) + else: + x = self.layers(x) else: x = self.layers(x) - if self.config["use_shake_drop"]: - alpha, beta = shake_get_alpha_beta(self.training, x.is_cuda) - bl = shake_drop_get_bl( - self.block_index, - 1 - self.config["max_shake_drop_probability"], - self.num_blocks, - self.training, - x.is_cuda - ) - x = shake_drop(x, alpha, beta, bl) + if self.config["use_skip_connection"]: + if self.config["multi_branch_choice"] == 'shake-drop': + alpha, beta = shake_get_alpha_beta(self.training, x.is_cuda) + bl = shake_drop_get_bl( + self.block_index, + 1 - self.config["max_shake_drop_probability"], + self.num_blocks, + self.training, + x.is_cuda, + ) + x = shake_drop(x, alpha, beta, bl) + + if self.config["use_skip_connection"]: + x = x + residual - x = x + residual return x diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py index 8fefa990c..59cd45d5d 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py @@ -5,7 +5,7 @@ from ConfigSpace.hyperparameters import ( CategoricalHyperparameter, UniformFloatHyperparameter, - UniformIntegerHyperparameter + UniformIntegerHyperparameter, ) import torch @@ -69,8 +69,8 @@ def build_backbone(self, input_shape: Tuple[int, ...]) -> torch.nn.Sequential: dropout=self.config[f'dropout_{i}'] if self.config['use_dropout'] else None ) ) - - layers.append(torch.nn.BatchNorm1d(self.config["num_units_%i" % self.config['num_groups']])) + if self.config['use_batch_norm']: + layers.append(torch.nn.BatchNorm1d(self.config["num_units_%i" % self.config['num_groups']])) backbone = torch.nn.Sequential(*layers) return backbone @@ -107,6 +107,18 @@ def get_hyperparameter_search_space( # type: ignore[override] value_range=(True, False), default_value=False, ), + use_batch_norm: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_batch_norm", + value_range=(True, False), + default_value=False, + ), + use_skip_connection: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_skip_connection", + value_range=(True, False), + default_value=True, + ), + multi_branch_choice: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="mb_choice", + value_range=('None', 'shake-shake', 'shake-drop'), + default_value='shake-drop', + ), max_units: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="max_units", value_range=(10, 1024), default_value=200), @@ -119,18 +131,11 @@ def get_hyperparameter_search_space( # type: ignore[override] max_dropout: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="max_dropout", value_range=(0, 0.8), default_value=0.5), - use_shake_shake: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_shake_shake", - value_range=(True, False), - default_value=True), - use_shake_drop: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_shake_drop", - value_range=(True, False), - default_value=True), max_shake_drop_probability: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="max_shake_drop_probability", value_range=(0, 1), default_value=0.5), ) -> ConfigurationSpace: - cs = ConfigurationSpace() # Support for different shapes @@ -141,23 +146,23 @@ def get_hyperparameter_search_space( # type: ignore[override] # repetitions is num_groups add_hyperparameter(cs, num_groups, UniformIntegerHyperparameter) add_hyperparameter(cs, blocks_per_group, UniformIntegerHyperparameter) - + add_hyperparameter(cs, max_units, UniformIntegerHyperparameter) add_hyperparameter(cs, activation, CategoricalHyperparameter) + # activation controlled batch normalization + add_hyperparameter(cs, use_batch_norm, CategoricalHyperparameter) add_hyperparameter(cs, output_dim, UniformIntegerHyperparameter) - use_shake_shake = get_hyperparameter(use_shake_shake, CategoricalHyperparameter) - use_shake_drop = get_hyperparameter(use_shake_drop, CategoricalHyperparameter) - shake_drop_prob = get_hyperparameter(max_shake_drop_probability, UniformFloatHyperparameter) - cs.add_hyperparameters([use_shake_shake, use_shake_drop, shake_drop_prob]) - cs.add_condition(CS.EqualsCondition(shake_drop_prob, use_shake_drop, True)) - - add_hyperparameter(cs, max_units, UniformIntegerHyperparameter) - use_dropout = get_hyperparameter(use_dropout, CategoricalHyperparameter) max_dropout = get_hyperparameter(max_dropout, UniformFloatHyperparameter) - - cs.add_hyperparameters([use_dropout]) - cs.add_hyperparameters([max_dropout]) cs.add_condition(CS.EqualsCondition(max_dropout, use_dropout, True)) + use_sc = get_hyperparameter(use_skip_connection, CategoricalHyperparameter) + mb_choice = get_hyperparameter(multi_branch_choice, CategoricalHyperparameter) + shake_drop_prob = get_hyperparameter(max_shake_drop_probability, UniformFloatHyperparameter) + cs.add_hyperparameters([use_sc, mb_choice, shake_drop_prob]) + cs.add_condition(CS.EqualsCondition(mb_choice, use_sc, True)) + # TODO check if shake_drop is as an option in mb_choice + # Incomplete work + cs.add_condition(CS.EqualsCondition(shake_drop_prob, mb_choice, "shake-drop")) + return cs diff --git a/autoPyTorch/pipeline/components/setup/optimizer/AdamOptimizer.py b/autoPyTorch/pipeline/components/setup/optimizer/AdamOptimizer.py index f86ea170b..2fef66aac 100644 --- a/autoPyTorch/pipeline/components/setup/optimizer/AdamOptimizer.py +++ b/autoPyTorch/pipeline/components/setup/optimizer/AdamOptimizer.py @@ -1,7 +1,9 @@ from typing import Any, Dict, Optional, Union +import ConfigSpace as CS from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( + CategoricalHyperparameter, UniformFloatHyperparameter, ) @@ -11,7 +13,7 @@ from autoPyTorch.datasets.base_dataset import BaseDatasetPropertiesType from autoPyTorch.pipeline.components.setup.optimizer.base_optimizer import BaseOptimizerComponent -from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter class AdamOptimizer(BaseOptimizerComponent): @@ -22,7 +24,8 @@ class AdamOptimizer(BaseOptimizerComponent): lr (float): learning rate (default: 1e-2) beta1 (float): coefficients used for computing running averages of gradient beta2 (float): coefficients used for computing running averages of square - weight_decay (float): weight decay (L2 penalty) + use_weight_decay (bool): flag for the activation of weight decay + weight_decay (float): weight decay (L2 penalty) (default: 0) random_state (Optional[np.random.RandomState]): random state """ @@ -31,13 +34,15 @@ def __init__( lr: float, beta1: float, beta2: float, - weight_decay: float, + use_weight_decay: bool, + weight_decay: float = 0, random_state: Optional[np.random.RandomState] = None, ): super().__init__() self.lr = lr self.beta1 = beta1 self.beta2 = beta2 + self.use_weight_decay = use_weight_decay self.weight_decay = weight_decay self.random_state = random_state @@ -87,6 +92,10 @@ def get_hyperparameter_search_space( beta2: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="beta2", value_range=(0.9, 0.9999), default_value=0.9), + use_weight_decay: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_weight_decay", + value_range=(True, False), + default_value=True, + ), weight_decay: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="weight_decay", value_range=(0.0, 0.1), default_value=0.0), @@ -97,6 +106,16 @@ def get_hyperparameter_search_space( add_hyperparameter(cs, lr, UniformFloatHyperparameter) add_hyperparameter(cs, beta1, UniformFloatHyperparameter) add_hyperparameter(cs, beta2, UniformFloatHyperparameter) - add_hyperparameter(cs, weight_decay, UniformFloatHyperparameter) + weight_decay = get_hyperparameter(weight_decay, UniformFloatHyperparameter) + use_weight_decay = get_hyperparameter(use_weight_decay, CategoricalHyperparameter) + cs.add_hyperparameters([use_weight_decay, weight_decay]) + + cs.add_condition( + CS.EqualsCondition( + weight_decay, + use_weight_decay, + True, + ) + ) return cs diff --git a/autoPyTorch/pipeline/components/setup/optimizer/AdamWOptimizer.py b/autoPyTorch/pipeline/components/setup/optimizer/AdamWOptimizer.py index 47ccc6e82..f7df85756 100644 --- a/autoPyTorch/pipeline/components/setup/optimizer/AdamWOptimizer.py +++ b/autoPyTorch/pipeline/components/setup/optimizer/AdamWOptimizer.py @@ -1,7 +1,9 @@ from typing import Any, Dict, Optional, Union +import ConfigSpace as CS from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( + CategoricalHyperparameter, UniformFloatHyperparameter, ) @@ -11,7 +13,7 @@ from autoPyTorch.datasets.base_dataset import BaseDatasetPropertiesType from autoPyTorch.pipeline.components.setup.optimizer.base_optimizer import BaseOptimizerComponent -from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter class AdamWOptimizer(BaseOptimizerComponent): @@ -22,7 +24,8 @@ class AdamWOptimizer(BaseOptimizerComponent): lr (float): learning rate (default: 1e-2) beta1 (float): coefficients used for computing running averages of gradient beta2 (float): coefficients used for computing running averages of square - weight_decay (float): weight decay (L2 penalty) + use_weight_decay (bool): flag for the activation of weight decay + weight_decay (float): weight decay (L2 penalty) (default: 0) random_state (Optional[np.random.RandomState]): random state """ @@ -31,13 +34,15 @@ def __init__( lr: float, beta1: float, beta2: float, - weight_decay: float, + use_weight_decay: bool, + weight_decay: float = 0, random_state: Optional[np.random.RandomState] = None, ): super().__init__() self.lr = lr self.beta1 = beta1 self.beta2 = beta2 + self.use_weight_decay = use_weight_decay self.weight_decay = weight_decay self.random_state = random_state @@ -87,6 +92,10 @@ def get_hyperparameter_search_space( beta2: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="beta2", value_range=(0.9, 0.9999), default_value=0.9), + use_weight_decay: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_weight_decay", + value_range=(True, False), + default_value=True, + ), weight_decay: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="weight_decay", value_range=(0.0, 0.1), default_value=0.0), @@ -97,6 +106,17 @@ def get_hyperparameter_search_space( add_hyperparameter(cs, lr, UniformFloatHyperparameter) add_hyperparameter(cs, beta1, UniformFloatHyperparameter) add_hyperparameter(cs, beta2, UniformFloatHyperparameter) - add_hyperparameter(cs, weight_decay, UniformFloatHyperparameter) + + weight_decay = get_hyperparameter(weight_decay, UniformFloatHyperparameter) + use_weight_decay = get_hyperparameter(use_weight_decay, CategoricalHyperparameter) + cs.add_hyperparameters([use_weight_decay, weight_decay]) + + cs.add_condition( + CS.EqualsCondition( + weight_decay, + use_weight_decay, + True, + ) + ) return cs diff --git a/autoPyTorch/pipeline/components/setup/optimizer/RMSpropOptimizer.py b/autoPyTorch/pipeline/components/setup/optimizer/RMSpropOptimizer.py index a64edc713..d1dc6f077 100644 --- a/autoPyTorch/pipeline/components/setup/optimizer/RMSpropOptimizer.py +++ b/autoPyTorch/pipeline/components/setup/optimizer/RMSpropOptimizer.py @@ -1,7 +1,9 @@ from typing import Any, Dict, Optional, Union +import ConfigSpace as CS from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( + CategoricalHyperparameter, UniformFloatHyperparameter, ) @@ -11,7 +13,7 @@ from autoPyTorch.datasets.base_dataset import BaseDatasetPropertiesType from autoPyTorch.pipeline.components.setup.optimizer.base_optimizer import BaseOptimizerComponent -from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter class RMSpropOptimizer(BaseOptimizerComponent): @@ -24,6 +26,7 @@ class RMSpropOptimizer(BaseOptimizerComponent): lr (float): learning rate (default: 1e-2) momentum (float): momentum factor (default: 0) alpha (float): smoothing constant (default: 0.99) + use_weight_decay (bool): flag for the activation of weight decay weight_decay (float): weight decay (L2 penalty) (default: 0) random_state (Optional[np.random.RandomState]): random state """ @@ -33,13 +36,15 @@ def __init__( lr: float, momentum: float, alpha: float, - weight_decay: float, + use_weight_decay: bool, + weight_decay: float = 0, random_state: Optional[np.random.RandomState] = None, ): super().__init__() self.lr = lr self.momentum = momentum self.alpha = alpha + self.use_weight_decay = use_weight_decay self.weight_decay = weight_decay self.random_state = random_state @@ -87,6 +92,10 @@ def get_hyperparameter_search_space( alpha: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="alpha", value_range=(0.1, 0.99), default_value=0.99), + use_weight_decay: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_weight_decay", + value_range=(True, False), + default_value=True, + ), weight_decay: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="weight_decay", value_range=(0.0, 0.1), default_value=0.0), @@ -100,6 +109,17 @@ def get_hyperparameter_search_space( add_hyperparameter(cs, lr, UniformFloatHyperparameter) add_hyperparameter(cs, alpha, UniformFloatHyperparameter) add_hyperparameter(cs, momentum, UniformFloatHyperparameter) - add_hyperparameter(cs, weight_decay, UniformFloatHyperparameter) + + weight_decay = get_hyperparameter(weight_decay, UniformFloatHyperparameter) + use_weight_decay = get_hyperparameter(use_weight_decay, CategoricalHyperparameter) + cs.add_hyperparameters([use_weight_decay, weight_decay]) + + cs.add_condition( + CS.EqualsCondition( + weight_decay, + use_weight_decay, + True, + ) + ) return cs diff --git a/autoPyTorch/pipeline/components/setup/optimizer/SGDOptimizer.py b/autoPyTorch/pipeline/components/setup/optimizer/SGDOptimizer.py index 2e34aeaf4..492bdf97e 100644 --- a/autoPyTorch/pipeline/components/setup/optimizer/SGDOptimizer.py +++ b/autoPyTorch/pipeline/components/setup/optimizer/SGDOptimizer.py @@ -1,7 +1,9 @@ from typing import Any, Dict, Optional, Union +import ConfigSpace as CS from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( + CategoricalHyperparameter, UniformFloatHyperparameter, ) @@ -11,7 +13,7 @@ from autoPyTorch.datasets.base_dataset import BaseDatasetPropertiesType from autoPyTorch.pipeline.components.setup.optimizer.base_optimizer import BaseOptimizerComponent -from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter class SGDOptimizer(BaseOptimizerComponent): @@ -21,21 +23,23 @@ class SGDOptimizer(BaseOptimizerComponent): Args: lr (float): learning rate (default: 1e-2) momentum (float): momentum factor (default: 0) + use_weight_decay (bool): flag for the activation of weight decay weight_decay (float): weight decay (L2 penalty) (default: 0) random_state (Optional[np.random.RandomState]): random state """ - def __init__( self, lr: float, momentum: float, - weight_decay: float, + use_weight_decay: bool, + weight_decay: float = 0, random_state: Optional[np.random.RandomState] = None, ): super().__init__() self.lr = lr self.momentum = momentum + self.use_weight_decay = use_weight_decay self.weight_decay = weight_decay self.random_state = random_state @@ -79,6 +83,10 @@ def get_hyperparameter_search_space( value_range=(1e-5, 1e-1), default_value=1e-2, log=True), + use_weight_decay: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_weight_decay", + value_range=(True, False), + default_value=True, + ), weight_decay: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="weight_decay", value_range=(0.0, 0.1), default_value=0.0), @@ -86,12 +94,22 @@ def get_hyperparameter_search_space( value_range=(0.0, 0.99), default_value=0.0), ) -> ConfigurationSpace: - cs = ConfigurationSpace() # The learning rate for the model add_hyperparameter(cs, lr, UniformFloatHyperparameter) add_hyperparameter(cs, momentum, UniformFloatHyperparameter) - add_hyperparameter(cs, weight_decay, UniformFloatHyperparameter) + + weight_decay = get_hyperparameter(weight_decay, UniformFloatHyperparameter) + use_weight_decay = get_hyperparameter(use_weight_decay, CategoricalHyperparameter) + cs.add_hyperparameters([use_weight_decay, weight_decay]) + + cs.add_condition( + CS.EqualsCondition( + weight_decay, + use_weight_decay, + True, + ) + ) return cs diff --git a/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py b/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py new file mode 100644 index 000000000..964d4b993 --- /dev/null +++ b/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py @@ -0,0 +1,158 @@ +import typing +from copy import deepcopy + +from ConfigSpace.configuration_space import ConfigurationSpace +from ConfigSpace.hyperparameters import ( + CategoricalHyperparameter, + UniformFloatHyperparameter, +) + +import numpy as np + +import torch + + +from autoPyTorch.constants import CLASSIFICATION_TASKS, STRING_TO_TASK_TYPES +from autoPyTorch.pipeline.components.training.trainer.base_trainer import BaseTrainerComponent +from autoPyTorch.utils.logging_ import PicklableClientLogger + + +class AdversarialTrainer(BaseTrainerComponent): + def __init__( + self, + epsilon: float, + weighted_loss: bool = False, + random_state: typing.Optional[np.random.RandomState] = None, + ): + """ + This class handles the training of a network for a single given epoch. + + Args: + epsilon (float): The perturbation magnitude. + + """ + super().__init__(random_state=random_state) + self.epsilon = epsilon + self.weighted_loss = weighted_loss + + def data_preparation(self, X: np.ndarray, y: np.ndarray, + ) -> typing.Tuple[typing.Tuple[np.ndarray, np.ndarray], typing.Dict[str, np.ndarray]]: + """Generate adversarial examples from the original inputs. + + Args: + X (np.ndarray): The batch training features + y (np.ndarray): The batch training labels + + Returns: + typing.Tuple[np.ndarray, np.ndarray]: original examples, adversarial examples. + typing.Dict[str, np.ndarray]: arguments to the criterion function. + """ + X_adversarial = self.fgsm_attack(X, y) + return (X, X_adversarial), {'y_a': y} + + def criterion_preparation(self, y_a: np.ndarray, y_b: np.ndarray = None, lam: float = 1.0 + ) -> typing.Callable: + # Initial implementation, consider the adversarial loss and the normal network loss + # equally. + return lambda criterion, pred, adversarial_pred: 0.5 * criterion(pred, y_a) + \ + 0.5 * criterion(adversarial_pred, y_a) + + def train_step(self, data: np.ndarray, targets: np.ndarray) -> Tuple[float, torch.Tensor]: + """ + Allows to train 1 step of gradient descent, given a batch of train/labels + + Args: + data (np.ndarray): input features to the network + targets (np.ndarray): ground truth to calculate loss + + Returns: + torch.Tensor: The predictions of the network + float: the loss incurred in the prediction + """ + # prepare + data = data.float().to(self.device) + targets = targets.long().to(self.device) + + data, criterion_kwargs = self.data_preparation(data, targets) + original_data = data[0] + adversarial_data = data[1] + + original_data = torch.autograd.Variable(original_data) + adversarial_data = torch.autograd.Variable(adversarial_data) + + # training + self.optimizer.zero_grad() + original_outputs = self.model(original_data) + adversarial_output = self.model(adversarial_data) + + loss_func = self.criterion_preparation(**criterion_kwargs) + loss = loss_func(self.criterion, original_outputs, adversarial_output) + loss.backward() + self.optimizer.step() + if self.scheduler: + if 'ReduceLROnPlateau' in self.scheduler.__class__.__name__: + self.scheduler.step(loss) + else: + self.scheduler.step() + # only passing the original outputs since we do not care about + # the adversarial performance. + return loss.item(), original_outputs + + def fgsm_attack( + self, + data: np.ndarray, + targets: np.ndarray, + ) -> np.ndarray: + """ + Generates the adversarial examples. + + Args: + data (np.ndarray): input features to the network + targets (np.ndarray): ground truth to calculate loss + + Returns: + adv_data (np.ndarray): the adversarial examples. + """ + data_copy = deepcopy(data) + data_copy = data_copy.float().to(self.device) + targets = targets.long().to(self.device) + data_copy = torch.autograd.Variable(data_copy) + data_copy.requires_grad = True + + outputs = self.model(data_copy) + cost = self.criterion(outputs, targets) + + grad = torch.autograd.grad(cost, data_copy, retain_graph=False, create_graph=False)[0] + + adv_data = data_copy + self.epsilon * grad.sign() + adv_data = torch.clamp(adv_data, min=0, max=1).detach() + + return adv_data + + @staticmethod + def get_properties(dataset_properties: typing.Optional[typing.Dict[str, typing.Any]] = None + ) -> typing.Dict[str, typing.Union[str, bool]]: + + return { + 'shortname': 'AdversarialTrainer', + 'name': 'AdversarialTrainer', + 'handles_tabular': True, + 'handles_image': False, + 'handles_time_series': False, + } + + @staticmethod + def get_hyperparameter_search_space(dataset_properties: typing.Optional[typing.Dict] = None, + epsilon: typing.Tuple[typing.Tuple[float, float], float] = ((0.05, 0.2), 0.2), + weighted_loss: typing.Tuple[typing.Tuple, bool] = ((True, False), True) + ) -> ConfigurationSpace: + epsilon = UniformFloatHyperparameter( + "epsilon", epsilon[0][0], epsilon[0][1], default_value=epsilon[1]) + weighted_loss = CategoricalHyperparameter("weighted_loss", choices=weighted_loss[0], + default_value=weighted_loss[1]) + cs = ConfigurationSpace() + cs.add_hyperparameters([epsilon]) + if dataset_properties is not None: + if STRING_TO_TASK_TYPES[dataset_properties['task_type']] not in CLASSIFICATION_TASKS: + cs.add_hyperparameters([weighted_loss]) + return cs diff --git a/autoPyTorch/pipeline/components/training/trainer/GridCutMixTrainer.py b/autoPyTorch/pipeline/components/training/trainer/GridCutMixTrainer.py new file mode 100644 index 000000000..fb22e7cb8 --- /dev/null +++ b/autoPyTorch/pipeline/components/training/trainer/GridCutMixTrainer.py @@ -0,0 +1,68 @@ +import typing + +import numpy as np + +import torch + +from autoPyTorch.pipeline.components.training.trainer.base_trainer import BaseTrainerComponent +from autoPyTorch.pipeline.components.training.trainer.mixup_utils import MixUp + + +class GridCutMixTrainer(MixUp, BaseTrainerComponent): + + def data_preparation(self, X: np.ndarray, y: np.ndarray, + ) -> typing.Tuple[np.ndarray, typing.Dict[str, np.ndarray]]: + """ + Depending on the trainer choice, data fed to the network might be pre-processed + on a different way. That is, in standard training we provide the data to the + network as we receive it to the loader. Some regularization techniques, like mixup + alter the data. + + Args: + X (np.ndarray): The batch training features + y (np.ndarray): The batch training labels + + Returns: + np.ndarray: that processes data + typing.Dict[str, np.ndarray]: arguments to the criterion function + """ + beta = 1.0 + lam = np.random.beta(beta, beta) + batch_size, channel, W, H = X.size() + index = torch.randperm(batch_size).cuda() if X.is_cuda else torch.randperm(batch_size) + + r = np.random.rand(1) + if beta <= 0 or r > self.alpha: + return X, {'y_a': y, 'y_b': y[index], 'lam': 1} + + # Draw parameters of a random bounding box + # Where to cut basically + cut_rat = np.sqrt(1. - lam) + cut_w = np.int(W * cut_rat) + cut_h = np.int(H * cut_rat) + cx = np.random.randint(W) + cy = np.random.randint(H) + bbx1 = np.clip(cx - cut_w // 2, 0, W) + bby1 = np.clip(cy - cut_h // 2, 0, H) + bbx2 = np.clip(cx + cut_w // 2, 0, W) + bby2 = np.clip(cy + cut_h // 2, 0, H) + + X[:, :, bbx1:bbx2, bby1:bby2] = X[index, :, bbx1:bbx2, bby1:bby2] + + # Adjust lam + lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (X.size()[-1] * X.size()[-2])) + + y_a, y_b = y, y[index] + + return X, {'y_a': y_a, 'y_b': y_b, 'lam': lam} + + @staticmethod + def get_properties(dataset_properties: typing.Optional[typing.Dict[str, typing.Any]] = None + ) -> typing.Dict[str, typing.Union[str, bool]]: + return { + 'shortname': 'GridCutMixTrainer', + 'name': 'GridCutMixTrainer', + 'handles_tabular': False, + 'handles_image': True, + 'handles_time_series': False, + } diff --git a/autoPyTorch/pipeline/components/training/trainer/GridCutOutTrainer.py b/autoPyTorch/pipeline/components/training/trainer/GridCutOutTrainer.py new file mode 100644 index 000000000..37c71d53b --- /dev/null +++ b/autoPyTorch/pipeline/components/training/trainer/GridCutOutTrainer.py @@ -0,0 +1,56 @@ +import typing + +import numpy as np + +from autoPyTorch.pipeline.components.training.trainer.base_trainer import BaseTrainerComponent +from autoPyTorch.pipeline.components.training.trainer.cutout_utils import CutOut + + +class GridCutOutTrainer(CutOut, BaseTrainerComponent): + + def data_preparation(self, X: np.ndarray, y: np.ndarray, + ) -> typing.Tuple[np.ndarray, typing.Dict[str, np.ndarray]]: + """ + Depending on the trainer choice, data fed to the network might be pre-processed + on a different way. That is, in standard training we provide the data to the + network as we receive it to the loader. Some regularization techniques, like mixup + alter the data. + + Args: + X (np.ndarray): The batch training features + y (np.ndarray): The batch training labels + + Returns: + np.ndarray: that processes data + typing.Dict[str, np.ndarray]: arguments to the criterion function + """ + r = np.random.rand(1) + batch_size, channel, W, H = X.size() + if r > self.cutout_prob: + return X, {'y_a': y, 'y_b': y, 'lam': 1} + + # Draw parameters of a random bounding box + # Where to cut basically + cut_rat = np.sqrt(1. - self.patch_ratio) + cut_w = np.int(W * cut_rat) + cut_h = np.int(H * cut_rat) + cx = np.random.randint(W) + cy = np.random.randint(H) + bbx1 = np.clip(cx - cut_w // 2, 0, W) + bby1 = np.clip(cy - cut_h // 2, 0, H) + bbx2 = np.clip(cx + cut_w // 2, 0, W) + bby2 = np.clip(cy + cut_h // 2, 0, H) + X[:, :, bbx1:bbx2, bby1:bby2] = 0.0 + + return X, {'y_a': y, 'y_b': y, 'lam': 1} + + @staticmethod + def get_properties(dataset_properties: typing.Optional[typing.Dict[str, typing.Any]] = None + ) -> typing.Dict[str, typing.Union[str, bool]]: + return { + 'shortname': 'GridCutOutTrainer', + 'name': 'GridCutOutTrainer', + 'handles_tabular': False, + 'handles_image': True, + 'handles_time_series': False, + } diff --git a/autoPyTorch/pipeline/components/training/trainer/MixUpTrainer.py b/autoPyTorch/pipeline/components/training/trainer/MixUpTrainer.py index 53ea09b1f..dc4910aeb 100644 --- a/autoPyTorch/pipeline/components/training/trainer/MixUpTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/MixUpTrainer.py @@ -1,22 +1,15 @@ -from typing import Callable, Dict, Optional, Tuple, Union - -from ConfigSpace.configuration_space import ConfigurationSpace -from ConfigSpace.hyperparameters import ( - CategoricalHyperparameter, - UniformFloatHyperparameter, -) +from typing import Any, Callable, Dict, Optional, Tuple, Union import numpy as np import torch -from autoPyTorch.constants import CLASSIFICATION_TASKS, STRING_TO_TASK_TYPES -from autoPyTorch.datasets.base_dataset import BaseDatasetPropertiesType from autoPyTorch.pipeline.components.training.trainer.base_trainer import BaseTrainerComponent +from autoPyTorch.pipeline.components.training.trainer.mixup_utils import MixUp from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter -class MixUpTrainer(BaseTrainerComponent): +class MixUpTrainer(MixUp, BaseTrainerComponent): """ References: Title: mixup: Beyond Empirical Risk Minimization @@ -24,21 +17,8 @@ class MixUpTrainer(BaseTrainerComponent): URL: https://arxiv.org/pdf/1710.09412.pdf%C2%A0 Github URL: https://github.com/facebookresearch/mixup-cifar10/blob/master/train.py#L119-L138 """ - def __init__(self, alpha: float, weighted_loss: bool = False, - random_state: Optional[np.random.RandomState] = None): - """ - This class handles the training of a network for a single given epoch. - - Args: - alpha (float): the mixup ratio - - """ - super().__init__(random_state=random_state) - self.weighted_loss = weighted_loss - self.alpha = alpha - - def data_preparation(self, X: torch.Tensor, y: torch.Tensor, - ) -> Tuple[torch.Tensor, Dict[str, np.ndarray]]: + def data_preparation(self, X: np.ndarray, y: np.ndarray, + ) -> Tuple[np.ndarray, Dict[str, np.ndarray]]: """ Depending on the trainer choice, data fed to the network might be pre-processed on a different way. That is, in standard training we provide the data to the @@ -52,7 +32,7 @@ def data_preparation(self, X: torch.Tensor, y: torch.Tensor, Returns: torch.Tensor: that processes data Dict[str, np.ndarray]: arguments to the criterion function - TODO: Fix this typing. It is not np.ndarray. + TODO: Fix this It is not np.ndarray. """ device = torch.device("cuda" if torch.cuda.is_available() else "cpu") @@ -64,32 +44,14 @@ def data_preparation(self, X: torch.Tensor, y: torch.Tensor, y_a, y_b = y, y[index] return mixed_x, {'y_a': y_a, 'y_b': y_b, 'lam': lam} - def criterion_preparation(self, y_a: torch.Tensor, y_b: torch.Tensor = None, lam: float = 1.0 - ) -> Callable: - return lambda criterion, pred: lam * criterion(pred, y_a) + (1 - lam) * criterion(pred, y_b) - @staticmethod - def get_properties(dataset_properties: Optional[Dict[str, BaseDatasetPropertiesType]] = None + def get_properties(dataset_properties: Optional[Dict[str, Any]] = None ) -> Dict[str, Union[str, bool]]: return { 'shortname': 'MixUpTrainer', 'name': 'MixUp Regularized Trainer', + 'handles_tabular': True, + 'handles_image': True, + 'handles_time_series': True, } - @staticmethod - def get_hyperparameter_search_space( - dataset_properties: Optional[Dict[str, BaseDatasetPropertiesType]] = None, - alpha: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="alpha", - value_range=(0, 1), - default_value=0.2), - weighted_loss: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="weighted_loss", - value_range=(True, False), - default_value=True), - ) -> ConfigurationSpace: - - cs = ConfigurationSpace() - add_hyperparameter(cs, alpha, UniformFloatHyperparameter) - if dataset_properties is not None: - if STRING_TO_TASK_TYPES[str(dataset_properties['task_type'])] in CLASSIFICATION_TASKS: - add_hyperparameter(cs, weighted_loss, CategoricalHyperparameter) - return cs diff --git a/autoPyTorch/pipeline/components/training/trainer/RowCutMixTrainer.py b/autoPyTorch/pipeline/components/training/trainer/RowCutMixTrainer.py new file mode 100644 index 000000000..b639156bb --- /dev/null +++ b/autoPyTorch/pipeline/components/training/trainer/RowCutMixTrainer.py @@ -0,0 +1,63 @@ +import random +import typing + +import numpy as np + +import torch + +from autoPyTorch.pipeline.components.training.trainer.base_trainer import BaseTrainerComponent +from autoPyTorch.pipeline.components.training.trainer.mixup_utils import MixUp + + +class RowCutMixTrainer(MixUp, BaseTrainerComponent): + + def data_preparation(self, X: np.ndarray, y: np.ndarray, + ) -> typing.Tuple[np.ndarray, typing.Dict[str, np.ndarray]]: + """ + Depending on the trainer choice, data fed to the network might be pre-processed + on a different way. That is, in standard training we provide the data to the + network as we receive it to the loader. Some regularization techniques, like mixup + alter the data. + + Args: + X (np.ndarray): The batch training features + y (np.ndarray): The batch training labels + + Returns: + np.ndarray: that processes data + typing.Dict[str, np.ndarray]: arguments to the criterion function + """ + beta = 1.0 + lam = np.random.beta(beta, beta) + batch_size = X.size()[0] + index = torch.randperm(batch_size).cuda() if X.is_cuda else torch.randperm(batch_size) + + r = np.random.rand(1) + if beta <= 0 or r > self.alpha: + return X, {'y_a': y, 'y_b': y[index], 'lam': 1} + + # The mixup component mixes up also on the batch dimension + # It is unlikely that the batch size is lower than the number of features, but + # be safe + size = min(X.shape[0], X.shape[1]) + indices = torch.tensor(random.sample(range(1, size), max(1, np.int(size * lam)))) + + X[:, indices] = X[index, :][:, indices] + + # Adjust lam + lam = 1 - ((len(indices)) / (X.size()[1])) + + y_a, y_b = y, y[index] + + return X, {'y_a': y_a, 'y_b': y_b, 'lam': lam} + + @staticmethod + def get_properties(dataset_properties: typing.Optional[typing.Dict[str, typing.Any]] = None + ) -> typing.Dict[str, typing.Union[str, bool]]: + return { + 'shortname': 'RowCutMixTrainer', + 'name': 'MixUp Regularized with Cutoff Tabular Trainer', + 'handles_tabular': True, + 'handles_image': False, + 'handles_time_series': False, + } diff --git a/autoPyTorch/pipeline/components/training/trainer/RowCutOutTrainer.py b/autoPyTorch/pipeline/components/training/trainer/RowCutOutTrainer.py new file mode 100644 index 000000000..660f6202f --- /dev/null +++ b/autoPyTorch/pipeline/components/training/trainer/RowCutOutTrainer.py @@ -0,0 +1,63 @@ +import random +import typing + +import numpy as np + +import torch + +from autoPyTorch.pipeline.components.training.trainer.base_trainer import BaseTrainerComponent +from autoPyTorch.pipeline.components.training.trainer.cutout_utils import CutOut + + +class RowCutOutTrainer(CutOut, BaseTrainerComponent): + + def data_preparation(self, X: np.ndarray, y: np.ndarray, + ) -> typing.Tuple[np.ndarray, typing.Dict[str, np.ndarray]]: + """ + Depending on the trainer choice, data fed to the network might be pre-processed + on a different way. That is, in standard training we provide the data to the + network as we receive it to the loader. Some regularization techniques, like mixup + alter the data. + + Args: + X (np.ndarray): The batch training features + y (np.ndarray): The batch training labels + + Returns: + np.ndarray: that processes data + typing.Dict[str, np.ndarray]: arguments to the criterion function + """ + + r = np.random.rand(1) + if r > self.cutout_prob: + y_a = y + y_b = y + lam = 1 + return X, {'y_a': y_a, 'y_b': y_b, 'lam': lam} + + # The mixup component mixes up also on the batch dimension + # It is unlikely that the batch size is lower than the number of features, but + # be safe + size = min(X.shape[0], X.shape[1]) + indices = torch.tensor(random.sample(range(1, size), max(1, np.int(size * self.patch_ratio)))) + + # We use an ordinal encoder on the tabular data + # -1 is the conceptual equivalent to 0 in a image, that does not + # have color as a feature and hence the network has to learn to deal + # without this data + X[:, indices.long()] = -1 + lam = 1 + y_a = y + y_b = y + return X, {'y_a': y_a, 'y_b': y_b, 'lam': lam} + + @staticmethod + def get_properties(dataset_properties: typing.Optional[typing.Dict[str, typing.Any]] = None + ) -> typing.Dict[str, typing.Union[str, bool]]: + return { + 'shortname': 'RowCutOutTrainer', + 'name': 'RowCutOutTrainer', + 'handles_tabular': True, + 'handles_image': False, + 'handles_time_series': False, + } diff --git a/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py b/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py index 33ec8f017..96ad5b190 100644 --- a/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py @@ -50,11 +50,14 @@ def criterion_preparation(self, y_a: torch.Tensor, y_b: torch.Tensor = None, lam return lambda criterion, pred: criterion(pred, y_a) @staticmethod - def get_properties(dataset_properties: Optional[Dict[str, BaseDatasetPropertiesType]] = None - ) -> Dict[str, Union[str, bool]]: + def get_properties(dataset_properties: typing.Optional[typing.Dict[str, typing.Any]] = None + ) -> typing.Dict[str, typing.Union[str, bool]]: return { 'shortname': 'StandardTrainer', - 'name': 'Standard Trainer', + 'name': 'StandardTrainer', + 'handles_tabular': True, + 'handles_image': True, + 'handles_time_series': True, } @staticmethod diff --git a/autoPyTorch/pipeline/components/training/trainer/__init__.py b/autoPyTorch/pipeline/components/training/trainer/__init__.py index 3134db201..71ba3ae2e 100755 --- a/autoPyTorch/pipeline/components/training/trainer/__init__.py +++ b/autoPyTorch/pipeline/components/training/trainer/__init__.py @@ -83,6 +83,68 @@ def __init__(self, def get_fit_requirements(self) -> Optional[List[FitRequirement]]: return self._fit_requirements + def get_available_components( + self, + dataset_properties: Optional[Dict[str, str]] = None, + include: Optional[List[str]] = None, + exclude: Optional[List[str]] = None, + ) -> Dict[str, autoPyTorchComponent]: + """ + Wrapper over get components to incorporate include/exclude + user specification + + Args: + dataset_properties (Optional[Dict[str, str]]): Describes the dataset to work on + include: Optional[Dict[str, Any]]: what components to include. It is an exhaustive + list, and will exclusively use this components. + exclude: Optional[Dict[str, Any]]: which components to skip + + Results: + Dict[str, autoPyTorchComponent]: A dictionary with valid components for this + choice object + + """ + if dataset_properties is None: + dataset_properties = {} + + if include is not None and exclude is not None: + raise ValueError( + "The argument include and exclude cannot be used together.") + + available_comp = self.get_components() + + if include is not None: + for incl in include: + if incl not in available_comp: + raise ValueError("Trying to include unknown component: " + "%s" % incl) + + components_dict = collections.OrderedDict() + for name in available_comp: + if include is not None and name not in include: + continue + elif exclude is not None and name in exclude: + continue + + # Allow training schemes exclusive for some task types + entry = available_comp[name] + task_type = dataset_properties['task_type'] + properties = entry.get_properties() + if 'tabular' in task_type and not properties['handles_tabular']: + continue + elif 'image' in task_type and not properties['handles_image']: + continue + elif 'time_series' in task_type and not properties['handles_time_series']: + continue + + if 'issparse' in dataset_properties: + if dataset_properties['issparse'] and \ + not available_comp[name].get_properties(dataset_properties)['handles_sparse']: + continue + components_dict[name] = available_comp[name] + + return components_dict + def get_components(self) -> Dict[str, autoPyTorchComponent]: """Returns the available trainer components diff --git a/autoPyTorch/pipeline/components/training/trainer/base_trainer.py b/autoPyTorch/pipeline/components/training/trainer/base_trainer.py index 0dba1e869..81f7e2732 100644 --- a/autoPyTorch/pipeline/components/training/trainer/base_trainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/base_trainer.py @@ -1,6 +1,8 @@ import time from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union +from ConfigSpace.configuration_space import ConfigurationSpace +from ConfigSpace.hyperparameters import CategoricalHyperparameter import numpy as np import pandas as pd @@ -14,6 +16,7 @@ from autoPyTorch.constants import FORECASTING_TASKS, REGRESSION_TASKS from autoPyTorch.pipeline.components.setup.lr_scheduler.constants import StepIntervalUnit +from autoPyTorch.constants import REGRESSION_TASKS, CLASSIFICATION_TASKS, STRING_TO_TASK_TYPES from autoPyTorch.pipeline.components.training.base_training import autoPyTorchTrainingComponent from autoPyTorch.pipeline.components.training.metrics.metrics import ( CLASSIFICATION_METRICS, @@ -199,7 +202,8 @@ def is_empty(self) -> bool: class BaseTrainerComponent(autoPyTorchTrainingComponent): - def __init__(self, random_state: Optional[np.random.RandomState] = None) -> None: + def __init__(self, weighted_loss: bool = False, + random_state: Optional[Union[np.random.RandomState, int]] = None) -> None: if random_state is None: # A trainer components need a random state for # sampling -- for example in MixUp training @@ -207,8 +211,7 @@ def __init__(self, random_state: Optional[np.random.RandomState] = None) -> None else: self.random_state = random_state super().__init__(random_state=self.random_state) - - self.weighted_loss: bool = False + self.weighted_loss = weighted_loss def prepare( self, @@ -485,4 +488,18 @@ def criterion_preparation(self, y_a: torch.Tensor, y_b: torch.Tensor = None, lam Returns: Callable: a lambda function that contains the new criterion calculation recipe """ - raise NotImplementedError + raise NotImplementedError() + + @staticmethod + def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, + weighted_loss: Tuple[Tuple, bool] = ((True, False), True), + use_swa: Tuple[Tuple, bool] = ((True, False), False) + ) -> ConfigurationSpace: + weighted_loss = CategoricalHyperparameter("weighted_loss", choices=weighted_loss[0], + default_value=weighted_loss[1]) + cs = ConfigurationSpace() + if dataset_properties is not None: + if STRING_TO_TASK_TYPES[dataset_properties['task_type']] not in CLASSIFICATION_TASKS: + cs.add_hyperparameters([weighted_loss]) + + return cs diff --git a/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py b/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py new file mode 100644 index 000000000..09d3c653b --- /dev/null +++ b/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py @@ -0,0 +1,62 @@ +import typing + +from ConfigSpace.configuration_space import ConfigurationSpace +from ConfigSpace.hyperparameters import ( + CategoricalHyperparameter, + UniformFloatHyperparameter, +) + +import numpy as np + +from autoPyTorch.constants import CLASSIFICATION_TASKS, STRING_TO_TASK_TYPES +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter + + +class CutOut: + def __init__(self, patch_ratio: float, + cutout_prob: float, + weighted_loss: bool = False, + random_state: typing.Optional[np.random.RandomState] = None): + """ + This class handles the training of a network for a single given epoch. + + Args: + patch_ratio (float): Defines the size of the cut off + cutout_prob (float): The probability of occurrence of this regulatization + + """ + self.weighted_loss = weighted_loss + self.patch_ratio = patch_ratio + self.cutout_prob = cutout_prob + self.random_state = random_state + + def criterion_preparation(self, y_a: np.ndarray, y_b: np.ndarray = None, lam: float = 1.0 + ) -> typing.Callable: + return lambda criterion, pred: lam * criterion(pred, y_a) + (1 - lam) * criterion(pred, y_b) + + @staticmethod + def get_hyperparameter_search_space( + dataset_properties: typing.Optional[typing.Dict] = None, + weighted_loss: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="weighted_loss", + value_range=(True, False), + default_value=True), + patch_ratio: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="alpha", + value_range=(0, 1), + default_value=0.2), + cutout_prob: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="alpha", + value_range=(0, 1), + default_value=0.2), + ) -> ConfigurationSpace: + + cs = ConfigurationSpace() + + add_hyperparameter(cs, patch_ratio, UniformFloatHyperparameter) + add_hyperparameter(cs, cutout_prob, UniformFloatHyperparameter) + if dataset_properties is not None: + if STRING_TO_TASK_TYPES[dataset_properties['task_type']] in CLASSIFICATION_TASKS: + add_hyperparameter(cs, weighted_loss, CategoricalHyperparameter) + + return cs diff --git a/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py b/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py new file mode 100644 index 000000000..297959356 --- /dev/null +++ b/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py @@ -0,0 +1,51 @@ +import typing + +from ConfigSpace.configuration_space import ConfigurationSpace +from ConfigSpace.hyperparameters import ( + CategoricalHyperparameter, + UniformFloatHyperparameter, +) + +import numpy as np + +from autoPyTorch.constants import CLASSIFICATION_TASKS, STRING_TO_TASK_TYPES +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter + + +class MixUp: + def __init__(self, alpha: float, + weighted_loss: bool = False, + random_state: typing.Optional[np.random.RandomState] = None): + """ + This class handles the training of a network for a single given epoch. + + Args: + alpha (float): the mixup ratio + + """ + self.weighted_loss = weighted_loss + self.alpha = alpha + self.random_state = random_state + + def criterion_preparation(self, y_a: np.ndarray, y_b: np.ndarray = None, lam: float = 1.0 + ) -> typing.Callable: + return lambda criterion, pred: lam * criterion(pred, y_a) + (1 - lam) * criterion(pred, y_b) + + @staticmethod + def get_hyperparameter_search_space( + dataset_properties: typing.Optional[typing.Dict] = None, + alpha: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="alpha", + value_range=(0, 1), + default_value=0.2), + weighted_loss: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="weighted_loss", + value_range=(True, False), + default_value=True), + ) -> ConfigurationSpace: + + cs = ConfigurationSpace() + add_hyperparameter(cs, alpha, UniformFloatHyperparameter) + if dataset_properties is not None: + if STRING_TO_TASK_TYPES[dataset_properties['task_type']] in CLASSIFICATION_TASKS: + add_hyperparameter(cs, weighted_loss, CategoricalHyperparameter) + + return cs diff --git a/test/test_pipeline/components/training/test_training.py b/test/test_pipeline/components/training/test_training.py index 6deda30ad..3ddbdacc5 100644 --- a/test/test_pipeline/components/training/test_training.py +++ b/test/test_pipeline/components/training/test_training.py @@ -21,10 +21,16 @@ ) from autoPyTorch.pipeline.components.training.trainer import ( TrainerChoice, +from autoPyTorch.pipeline.components.training.trainer.AdversarialTrainer import ( + AdversarialTrainer ) +from autoPyTorch.pipeline.components.training.trainer.GridCutMixTrainer import GridCutMixTrainer +from autoPyTorch.pipeline.components.training.trainer.GridCutOutTrainer import GridCutOutTrainer from autoPyTorch.pipeline.components.training.trainer.MixUpTrainer import ( MixUpTrainer ) +from autoPyTorch.pipeline.components.training.trainer.RowCutMixTrainer import RowCutMixTrainer +from autoPyTorch.pipeline.components.training.trainer.RowCutOutTrainer import RowCutOutTrainer from autoPyTorch.pipeline.components.training.trainer.StandardTrainer import ( StandardTrainer ) @@ -346,80 +352,140 @@ def test_classification_epoch_training(self, n_samples): if counter > epochs: pytest.fail(f"Could not overfit a dummy classification under {epochs} epochs") - -class TestTrainer(unittest.TestCase): - def test_every_trainer_is_valid(self): - """ - Makes sure that every trainer is a valid estimator. - That is, we can fully create an object via get/set params. - - This also test that we can properly initialize each one - of them - """ - trainer_choice = TrainerChoice(dataset_properties={}) - - # Make sure all components are returned - self.assertEqual(len(trainer_choice.get_components().keys()), 2) - - # For every optimizer in the components, make sure - # that it complies with the scikit learn estimator. - # This is important because usually components are forked to workers, - # so the set/get params methods should recreate the same object - for name, trainer in trainer_choice.get_components().items(): - config = trainer.get_hyperparameter_search_space().sample_configuration() - estimator = trainer(**config) - estimator_clone = clone(estimator) - estimator_clone_params = estimator_clone.get_params() - - # Make sure all keys are copied properly - for k in estimator.get_params().keys(): - self.assertIn(k, estimator_clone_params) - - # Make sure the params getter of estimator are honored - klass = estimator.__class__ - new_object_params = estimator.get_params(deep=False) - for name, param in new_object_params.items(): - new_object_params[name] = clone(param, safe=False) - new_object = klass(**new_object_params) - params_set = new_object.get_params(deep=False) - - for name in new_object_params: - param1 = new_object_params[name] - param2 = params_set[name] - self.assertEqual(param1, param2) - - def test_get_set_config_space(self): - """Make sure that we can setup a valid choice in the trainer - choice""" - trainer_choice = TrainerChoice(dataset_properties={'task_type': 'tabular_classification'}) - cs = trainer_choice.get_hyperparameter_search_space() - - # Make sure that all hyperparameters are part of the serach space - self.assertListEqual( - sorted(cs.get_hyperparameter('__choice__').choices), - sorted(list(trainer_choice.get_components().keys())) - ) - - # Make sure we can properly set some random configs - # Whereas just one iteration will make sure the algorithm works, - # doing five iterations increase the confidence. We will be able to - # catch component specific crashes - for _ in range(5): - config = cs.sample_configuration() - config_dict = copy.deepcopy(config.get_dictionary()) - trainer_choice.set_hyperparameters(config) - - self.assertEqual(trainer_choice.choice.__class__, - trainer_choice.get_components()[config_dict['__choice__']]) - - # Then check the choice configuration - selected_choice = config_dict.pop('__choice__', None) - for key, value in config_dict.items(): - # Remove the selected_choice string from the parameter - # so we can query in the object for it - key = key.replace(selected_choice + ':', '') - self.assertIn(key, vars(trainer_choice.choice)) - self.assertEqual(value, trainer_choice.choice.__dict__[key]) +def test_every_trainer_is_valid(): + """ + Makes sure that every trainer is a valid estimator. + That is, we can fully create an object via get/set params. + + This also test that we can properly initialize each one + of them + """ + trainer_choice = TrainerChoice(dataset_properties={}) + + # Make sure all components are returned + assert len(trainer_choice.get_components().keys()) == 6 + + # For every optimizer in the components, make sure + # that it complies with the scikit learn estimator. + # This is important because usually components are forked to workers, + # so the set/get params methods should recreate the same object + for name, trainer in trainer_choice.get_components().items(): + config = trainer.get_hyperparameter_search_space().sample_configuration() + estimator = trainer(**config) + estimator_clone = clone(estimator) + estimator_clone_params = estimator_clone.get_params() + + # Make sure all keys are copied properly + for k, v in estimator.get_params().items(): + assert k in estimator_clone_params + + # Make sure the params getter of estimator are honored + klass = estimator.__class__ + new_object_params = estimator.get_params(deep=False) + for name, param in new_object_params.items(): + new_object_params[name] = clone(param, safe=False) + new_object = klass(**new_object_params) + params_set = new_object.get_params(deep=False) + + for name in new_object_params: + param1 = new_object_params[name] + param2 = params_set[name] + assert param1 == param2 + + +@pytest.mark.parametrize("test_input,expected", [ + ("tabular_classification", set(['RowCutMixTrainer', 'RowCutOutTrainer'])), + ("image_classification", set(['GridCutMixTrainer', 'GridCutOutTrainer'])), + ("time_series_classification", set([])), +]) +def test_get_set_config_space(test_input, expected): + """Make sure that we can setup a valid choice in the trainer + choice""" + trainer_choice = TrainerChoice(dataset_properties={'task_type': test_input}) + cs = trainer_choice.get_hyperparameter_search_space() + + # Make sure that all hyperparameters are part of the serach space + # Filtering out the ones not supported for the given task + always_expected_trainers = set(['StandardTrainer', 'MixUpTrainer']) + assert set(cs.get_hyperparameter('__choice__').choices) == always_expected_trainers | expected + + # Make sure we can properly set some random configs + # Whereas just one iteration will make sure the algorithm works, + # doing five iterations increase the confidence. We will be able to + # catch component specific crashes + for i in range(5): + config = cs.sample_configuration() + config_dict = copy.deepcopy(config.get_dictionary()) + trainer_choice.set_hyperparameters(config) + + assert trainer_choice.choice.__class__ == trainer_choice.get_components( + )[config_dict['__choice__']] + + # Then check the choice configuration + selected_choice = config_dict.pop('__choice__', None) + for key, value in config_dict.items(): + # Remove the selected_choice string from the parameter + # so we can query in the object for it + key = key.replace(selected_choice + ':', '') + assert key in vars(trainer_choice.choice) + assert value == trainer_choice.choice.__dict__[key] + + +@pytest.mark.parametrize("cutmix_prob", [1.0, 0.0]) +@pytest.mark.parametrize("regularizer,X", [ + (GridCutMixTrainer, torch.from_numpy(np.full(shape=(2, 3, 10, 12), fill_value=255))), + (RowCutMixTrainer, torch.from_numpy(np.array([[1, 2, 3, 4, 5], [6, 7, 8, 9, 10]]))), +]) +def test_mixup_regularizers(cutmix_prob, regularizer, X): + trainer = regularizer(cutmix_prob) + + def criterion(a, b): + return (a == b).sum() + + y = torch.from_numpy(np.array([[1], [0]])) + y_pred = torch.from_numpy(np.array([[1], [1]])) + X_new, target_dict = trainer.data_preparation(X, y) + loss_func = trainer.criterion_preparation(**target_dict) + if cutmix_prob == 0.0: + # we do not expect a change to the data + np.testing.assert_array_equal(X_new.numpy(), X.numpy()) + assert target_dict['lam'] == 1 + # No mixup but a plain criterion, which as seen above is + # a sum of matches, that is, a integer + assert isinstance(loss_func(criterion, y_pred).numpy().item(), int) + else: + # There has to be a change in the features + np.any(np.not_equal(X_new.numpy(), X.numpy())) + assert 0 < target_dict['lam'] < 1 + # There has to be a mixup of loss function + # That's why the loss function returns a float + + +@pytest.mark.parametrize("cutout_prob", [1.0, 0.0]) +@pytest.mark.parametrize("regularizer,X", [ + (GridCutOutTrainer, torch.from_numpy(np.full(shape=(2, 3, 10, 12), fill_value=255))), + (RowCutOutTrainer, torch.from_numpy(np.array([[1, 2, 3, 4, 5], [6, 7, 8, 9, 10]]))), +]) +def test_cutput_regularizers(cutout_prob, regularizer, X): + trainer = regularizer(cutout_prob=cutout_prob, patch_ratio=0.5) + + y = torch.from_numpy(np.array([[1], [0]])) + X_new, target_dict = trainer.data_preparation(X, y) + + # No mixing needed + assert target_dict['lam'] == 1 + if cutout_prob == 0.0: + # we do not expect a change to the data + np.testing.assert_array_equal(X_new.numpy(), X.numpy()) + else: + # There has to be a change in the features + if len(X.shape) > 2: + expected = 0.0 + else: + expected = -1 + # The original X does not have the expected value + # If a cutoff happened, then this value is gonna be there + assert expected in X_new def test_early_stopping(): @@ -470,5 +536,84 @@ def dummy_performance(*args, **kwargs): shutil.rmtree(fit_dictionary['backend'].temporary_directory) +class AdversarialTrainerTest(BaseTraining, unittest.TestCase): + + def test_epoch_training(self): + """ + Makes sure we are able to train a model and produce good + training performance + """ + trainer = AdversarialTrainer(epsilon=0.07) + trainer.prepare( + scheduler=None, + model=self.model, + metrics=self.metrics, + criterion=self.criterion, + budget_tracker=self.budget_tracker, + optimizer=self.optimizer, + device=self.device, + metrics_during_training=True, + task_type=self.task_type, + output_type=self.output_type, + labels=self.y + ) + + # Train the model + counter = 0 + accuracy = 0 + while accuracy < 0.7: + loss, metrics = trainer.train_epoch(self.loader, epoch=1, logger=self.logger, writer=None) + counter += 1 + accuracy = metrics['accuracy'] + + if counter > 1000: + self.fail("Could not overfit a dummy binary classification under 1000 epochs") + + +<<<<<<< HEAD +======= +class TrainerTest(unittest.TestCase): + def test_every_trainer_is_valid(self): + """ + Makes sure that every trainer is a valid estimator. + That is, we can fully create an object via get/set params. + + This also test that we can properly initialize each one + of them + """ + trainer_choice = TrainerChoice(dataset_properties={}) + + # Make sure all components are returned + self.assertEqual(len(trainer_choice.get_components().keys()), 7) + + # For every optimizer in the components, make sure + # that it complies with the scikit learn estimator. + # This is important because usually components are forked to workers, + # so the set/get params methods should recreate the same object + for name, trainer in trainer_choice.get_components().items(): + config = trainer.get_hyperparameter_search_space().sample_configuration() + estimator = trainer(**config) + estimator_clone = clone(estimator) + estimator_clone_params = estimator_clone.get_params() + + # Make sure all keys are copied properly + for k, v in estimator.get_params().items(): + self.assertIn(k, estimator_clone_params) + + # Make sure the params getter of estimator are honored + klass = estimator.__class__ + new_object_params = estimator.get_params(deep=False) + for name, param in new_object_params.items(): + new_object_params[name] = clone(param, safe=False) + new_object = klass(**new_object_params) + params_set = new_object.get_params(deep=False) + + for name in new_object_params: + param1 = new_object_params[name] + param2 = params_set[name] + self.assertEqual(param1, param2) + + +>>>>>>> Removing duplicate unit test if __name__ == '__main__': unittest.main() From 3b4feae4325bd93a7cd28113e537e70dda3be180 Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Fri, 19 Feb 2021 12:26:10 +0100 Subject: [PATCH 02/50] Add cyclic property to lr scheduler and use_swa to trainer swa working, se in progress Fixed bug in update model with swa model, add predict with snapshot ensemble; todo: add tests for both --- .../setup/lr_scheduler/CosineAnnealingLR.py | 1 + .../CosineAnnealingWarmRestarts.py | 1 + .../components/setup/lr_scheduler/CyclicLR.py | 3 +- .../setup/lr_scheduler/ExponentialLR.py | 3 +- .../setup/lr_scheduler/NoScheduler.py | 1 + .../setup/lr_scheduler/ReduceLROnPlateau.py | 1 + .../components/setup/lr_scheduler/StepLR.py | 1 + .../setup/lr_scheduler/base_scheduler.py | 3 +- .../components/setup/network/base_network.py | 36 +++++++---- .../setup/network_backbone/ResNetBackbone.py | 2 +- .../training/trainer/MixUpTrainer.py | 3 - .../training/trainer/StandardTrainer.py | 24 ++++++- .../components/training/trainer/__init__.py | 15 ++++- .../training/trainer/base_trainer.py | 64 ++++++++++++++++++- .../components/training/trainer/utils.py | 27 ++++++++ 15 files changed, 161 insertions(+), 24 deletions(-) create mode 100644 autoPyTorch/pipeline/components/training/trainer/utils.py diff --git a/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingLR.py b/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingLR.py index 12040178a..1b351ca89 100644 --- a/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingLR.py +++ b/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingLR.py @@ -61,6 +61,7 @@ def get_properties(dataset_properties: Optional[Dict[str, BaseDatasetPropertiesT return { 'shortname': 'CosineAnnealing', 'name': 'Cosine Annealing', + 'cyclic': False } @staticmethod diff --git a/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingWarmRestarts.py b/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingWarmRestarts.py index 894d532dd..ccb58b61d 100644 --- a/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingWarmRestarts.py +++ b/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingWarmRestarts.py @@ -69,6 +69,7 @@ def get_properties(dataset_properties: Optional[Dict[str, BaseDatasetPropertiesT return { 'shortname': 'CosineAnnealingWarmRestarts', 'name': 'Cosine Annealing WarmRestarts', + 'cyclic': True } @staticmethod diff --git a/autoPyTorch/pipeline/components/setup/lr_scheduler/CyclicLR.py b/autoPyTorch/pipeline/components/setup/lr_scheduler/CyclicLR.py index d26d3d495..35514145c 100644 --- a/autoPyTorch/pipeline/components/setup/lr_scheduler/CyclicLR.py +++ b/autoPyTorch/pipeline/components/setup/lr_scheduler/CyclicLR.py @@ -85,7 +85,8 @@ def get_properties(dataset_properties: Optional[Dict[str, BaseDatasetPropertiesT ) -> Dict[str, Union[str, bool]]: return { 'shortname': 'CyclicLR', - 'name': 'Cyclic Learning Rate Scheduler', + 'name': 'CyclicLR', + 'cyclic': True } @staticmethod diff --git a/autoPyTorch/pipeline/components/setup/lr_scheduler/ExponentialLR.py b/autoPyTorch/pipeline/components/setup/lr_scheduler/ExponentialLR.py index dc57cfc1e..ca89ec553 100644 --- a/autoPyTorch/pipeline/components/setup/lr_scheduler/ExponentialLR.py +++ b/autoPyTorch/pipeline/components/setup/lr_scheduler/ExponentialLR.py @@ -61,7 +61,8 @@ def get_properties(dataset_properties: Optional[Dict[str, BaseDatasetPropertiesT ) -> Dict[str, Union[str, bool]]: return { 'shortname': 'ExponentialLR', - 'name': 'Exponential Learning Rate Scheduler', + 'name': 'ExponentialLR', + 'cyclic': False } @staticmethod diff --git a/autoPyTorch/pipeline/components/setup/lr_scheduler/NoScheduler.py b/autoPyTorch/pipeline/components/setup/lr_scheduler/NoScheduler.py index 5a1f2e571..c91c73ae0 100644 --- a/autoPyTorch/pipeline/components/setup/lr_scheduler/NoScheduler.py +++ b/autoPyTorch/pipeline/components/setup/lr_scheduler/NoScheduler.py @@ -45,6 +45,7 @@ def get_properties(dataset_properties: Optional[Dict[str, BaseDatasetPropertiesT return { 'shortname': 'NoScheduler', 'name': 'No LR Scheduling', + 'cyclic': False } @staticmethod diff --git a/autoPyTorch/pipeline/components/setup/lr_scheduler/ReduceLROnPlateau.py b/autoPyTorch/pipeline/components/setup/lr_scheduler/ReduceLROnPlateau.py index ae87bfdd2..4eda659d8 100644 --- a/autoPyTorch/pipeline/components/setup/lr_scheduler/ReduceLROnPlateau.py +++ b/autoPyTorch/pipeline/components/setup/lr_scheduler/ReduceLROnPlateau.py @@ -81,6 +81,7 @@ def get_properties(dataset_properties: Optional[Dict[str, BaseDatasetPropertiesT return { 'shortname': 'ReduceLROnPlateau', 'name': 'ReduceLROnPlateau', + 'cyclic': False } @staticmethod diff --git a/autoPyTorch/pipeline/components/setup/lr_scheduler/StepLR.py b/autoPyTorch/pipeline/components/setup/lr_scheduler/StepLR.py index 1917e61ae..294191c8f 100644 --- a/autoPyTorch/pipeline/components/setup/lr_scheduler/StepLR.py +++ b/autoPyTorch/pipeline/components/setup/lr_scheduler/StepLR.py @@ -68,6 +68,7 @@ def get_properties(dataset_properties: Optional[Dict[str, BaseDatasetPropertiesT return { 'shortname': 'StepLR', 'name': 'StepLR', + 'cyclic': False } @staticmethod diff --git a/autoPyTorch/pipeline/components/setup/lr_scheduler/base_scheduler.py b/autoPyTorch/pipeline/components/setup/lr_scheduler/base_scheduler.py index e31f09475..671a70f6a 100644 --- a/autoPyTorch/pipeline/components/setup/lr_scheduler/base_scheduler.py +++ b/autoPyTorch/pipeline/components/setup/lr_scheduler/base_scheduler.py @@ -45,7 +45,8 @@ def transform(self, X: Dict[str, Any]) -> Dict[str, Any]: X.update( lr_scheduler=self.scheduler, - step_interval=self.step_interval + step_interval=self.step_interval, + is_cyclic_scheduler= self.get_properties()['cyclic'] ) return X diff --git a/autoPyTorch/pipeline/components/setup/network/base_network.py b/autoPyTorch/pipeline/components/setup/network/base_network.py index 768d0eb20..9970371a5 100644 --- a/autoPyTorch/pipeline/components/setup/network/base_network.py +++ b/autoPyTorch/pipeline/components/setup/network/base_network.py @@ -1,4 +1,5 @@ from typing import Any, Dict, Optional, Union +from typing import Any, Dict, List, Optional from ConfigSpace.configuration_space import ConfigurationSpace @@ -25,6 +26,9 @@ def __init__( random_state: Optional[np.random.RandomState] = None ) -> None: super(NetworkComponent, self).__init__() + + self.network = network + self.network_snapshots: List[torch.nn.Module] = [] self.random_state = random_state self.device = None self.add_fit_requirements([ @@ -69,7 +73,8 @@ def transform(self, X: Dict[str, Any]) -> Dict[str, Any]: """ The transform function updates the network in the X dictionary. """ - X.update({'network': self.network}) + X.update({'network': self.network, + 'network_snapshots': self.network_snapshots}) return X def get_network(self) -> nn.Module: @@ -108,24 +113,33 @@ def predict(self, loader: torch.utils.data.DataLoader) -> torch.Tensor: """ Performs batched prediction given a loader object """ - assert self.network is not None - self.network.eval() + if len(self.network_snapshots) == 0: + assert self.network is not None + return self._predict(network=self.network, loader=loader).cpu().numpy() + else: + # if there are network snapshots, + # take average of predictions of all snapshots + Y_snapshot_preds = list() + for network in self.network_snapshots: + Y_snapshot_preds.append(self._predict(network, loader)) + Y_snapshot_preds = torch.stack(Y_snapshot_preds) + return Y_snapshot_preds.mean(dim=0).cpu().numpy() + + def _predict(self, network: torch.nn.Module, loader: torch.utils.data.DataLoader) -> torch.Tensor: + network.eval() # Batch prediction Y_batch_preds = list() for i, (X_batch, Y_batch) in enumerate(loader): # Predict on batch X_batch = X_batch.float().to(self.device) + Y_batch_pred = network(X_batch).detach().cpu() + if self.final_activation is not None: + Y_batch_pred = self.final_activation(Y_batch_pred) + Y_batch_preds.append(Y_batch_pred) - with torch.no_grad(): - Y_batch_pred = self.network(X_batch) - if self.final_activation is not None: - Y_batch_pred = self.final_activation(Y_batch_pred) - - Y_batch_preds.append(Y_batch_pred.cpu()) - - return torch.cat(Y_batch_preds, 0).cpu().numpy() + return torch.cat(Y_batch_preds, 0) @staticmethod def get_hyperparameter_search_space(dataset_properties: Optional[Dict[str, BaseDatasetPropertiesType]] = None, diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py index 8cb6d5260..8ee3ed19b 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py @@ -163,7 +163,7 @@ def get_hyperparameter_search_space( # better generalization use_dropout = get_hyperparameter(use_dropout, CategoricalHyperparameter) cs.add_hyperparameters([use_dropout]) - + use_sc = get_hyperparameter(use_skip_connection, CategoricalHyperparameter) mb_choice = get_hyperparameter(multi_branch_choice, CategoricalHyperparameter) shake_drop_prob = get_hyperparameter(max_shake_drop_probability, UniformFloatHyperparameter) diff --git a/autoPyTorch/pipeline/components/training/trainer/MixUpTrainer.py b/autoPyTorch/pipeline/components/training/trainer/MixUpTrainer.py index dc4910aeb..7cf443126 100644 --- a/autoPyTorch/pipeline/components/training/trainer/MixUpTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/MixUpTrainer.py @@ -1,12 +1,10 @@ from typing import Any, Callable, Dict, Optional, Tuple, Union import numpy as np - import torch from autoPyTorch.pipeline.components.training.trainer.base_trainer import BaseTrainerComponent from autoPyTorch.pipeline.components.training.trainer.mixup_utils import MixUp -from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter class MixUpTrainer(MixUp, BaseTrainerComponent): @@ -54,4 +52,3 @@ def get_properties(dataset_properties: Optional[Dict[str, Any]] = None 'handles_image': True, 'handles_time_series': True, } - diff --git a/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py b/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py index 96ad5b190..3c0535a3a 100644 --- a/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py @@ -1,8 +1,14 @@ from typing import Callable, Dict, Optional, Tuple, Union from ConfigSpace.configuration_space import ConfigurationSpace +<<<<<<< HEAD +from ConfigSpace.hyperparameters import ( + CategoricalHyperparameter, +) +======= from ConfigSpace.hyperparameters import CategoricalHyperparameter +>>>>>>> swa working, se in progress import numpy as np import torch @@ -15,7 +21,10 @@ class StandardTrainer(BaseTrainerComponent): def __init__(self, weighted_loss: bool = False, - random_state: Optional[np.random.RandomState] = None): + use_swa: bool = False, + use_se: bool = False, + se_lastk: int = 3, + random_state: typing.Optional[np.random.RandomState] = None): """ This class handles the training of a network for a single given epoch. @@ -23,8 +32,11 @@ def __init__(self, weighted_loss: bool = False, weighted_loss (bool): whether to use weighted loss """ - super().__init__(random_state=random_state) - self.weighted_loss = weighted_loss + super().__init__(random_state=random_state, + weighted_loss=weighted_loss, + use_swa=use_swa, + use_se=use_se, + se_lastk=se_lastk) def data_preparation(self, X: torch.Tensor, y: torch.Tensor, ) -> Tuple[torch.Tensor, Dict[str, np.ndarray]]: @@ -66,6 +78,12 @@ def get_hyperparameter_search_space( weighted_loss: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="weighted_loss", value_range=(True, False), default_value=True), + use_swa: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_swa", + value_range=(True, False), + default_value=True), + use_se: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_se", + value_range=(True, False), + default_value=True), ) -> ConfigurationSpace: cs = ConfigurationSpace() if dataset_properties is not None: diff --git a/autoPyTorch/pipeline/components/training/trainer/__init__.py b/autoPyTorch/pipeline/components/training/trainer/__init__.py index 71ba3ae2e..f45f832c6 100755 --- a/autoPyTorch/pipeline/components/training/trainer/__init__.py +++ b/autoPyTorch/pipeline/components/training/trainer/__init__.py @@ -14,7 +14,7 @@ import numpy as np import torch -from torch.optim import Optimizer +from torch.optim import Optimizer, swa_utils from torch.optim.lr_scheduler import _LRScheduler from torch.utils.tensorboard.writer import SummaryWriter @@ -28,6 +28,7 @@ ) from autoPyTorch.pipeline.components.training.losses import get_loss from autoPyTorch.pipeline.components.training.metrics.utils import get_metrics +from autoPyTorch.pipeline.components.training.trainer.utils import update_model_state_dict_from_swa from autoPyTorch.pipeline.components.training.trainer.base_trainer import ( BaseTrainerComponent, BudgetTracker, @@ -275,6 +276,10 @@ def fit(self, X: Dict[str, Any], y: Any = None, **kwargs: Any) -> autoPyTorchCom y=y, **kwargs ) + # Add snapshots to base network to enable + # predicting with snapshot ensemble + if self.choice.use_se: + X['network_snapshots'].extend(self.choice.model_snapshots) return cast(autoPyTorchComponent, self.choice) @@ -426,6 +431,14 @@ def _fit(self, X: Dict[str, Any], y: Any = None, **kwargs: Any) -> 'TrainerChoic if self.run_summary.is_empty(): raise RuntimeError("Budget exhausted without finishing an epoch.") + if self.choice.use_swa: + # update batch norm statistics + swa_utils.update_bn(X['train_data_loader'], self.choice.swa_model) + # change model + update_model_state_dict_from_swa(X['network'], self.choice.swa_model.state_dict()) + if self.choice.use_se: + for model in self.choice.model_snapshots: + swa_utils.update_bn(X['train_data_loader'], model) # wrap up -- add score if not evaluating every epoch if not self.eval_valid_each_epoch(X): diff --git a/autoPyTorch/pipeline/components/training/trainer/base_trainer.py b/autoPyTorch/pipeline/components/training/trainer/base_trainer.py index 81f7e2732..b04e42bca 100644 --- a/autoPyTorch/pipeline/components/training/trainer/base_trainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/base_trainer.py @@ -1,6 +1,15 @@ +from copy import deepcopy +from queue import LifoQueue import time from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union +from ConfigSpace.conditions import EqualsCondition +from ConfigSpace.configuration_space import ConfigurationSpace +from ConfigSpace.hyperparameters import ( + CategoricalHyperparameter, + Constant +) + from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import CategoricalHyperparameter import numpy as np @@ -10,7 +19,7 @@ from sklearn.utils import check_random_state import torch -from torch.optim import Optimizer +from torch.optim import Optimizer, swa_utils from torch.optim.lr_scheduler import _LRScheduler from torch.utils.tensorboard.writer import SummaryWriter @@ -24,6 +33,7 @@ REGRESSION_METRICS, ) from autoPyTorch.pipeline.components.training.metrics.utils import calculate_score +from autoPyTorch.utils.common import FitRequirement from autoPyTorch.utils.implementations import get_loss_weight_strategy @@ -203,6 +213,9 @@ def is_empty(self) -> bool: class BaseTrainerComponent(autoPyTorchTrainingComponent): def __init__(self, weighted_loss: bool = False, + use_swa: bool = False, + use_se: bool = False, + se_lastk: int = 3, random_state: Optional[Union[np.random.RandomState, int]] = None) -> None: if random_state is None: # A trainer components need a random state for @@ -212,6 +225,12 @@ def __init__(self, weighted_loss: bool = False, self.random_state = random_state super().__init__(random_state=self.random_state) self.weighted_loss = weighted_loss + self.use_swa = use_swa + self.use_se = use_se + self.se_lastk = se_lastk + self.add_fit_requirements([ + FitRequirement("is_cyclic_scheduler", (bool,), user_defined=False, dataset_property=False), + ]) def prepare( self, @@ -245,6 +264,17 @@ def prepare( # setup the model self.model = model.to(device) + # in case we are using swa, maintain an averaged model, + if self.use_swa: + self.swa_model = swa_utils.AveragedModel(self.model) + + # in case we are using se or swa, initialise budget_threshold to know when to start swa or se + self._budget_threshold: int = int(0.75 * budget_tracker.max_epochs) + + # in case we are using se, initialise list to store model snapshots + if self.use_se: + self.model_snapshots: List[torch.nn.Module] = list() + # setup the optimizers self.optimizer = optimizer @@ -276,6 +306,24 @@ def on_epoch_end(self, X: Dict[str, Any], epoch: int) -> bool: If returns True, the training is stopped """ + if X['is_cyclic_scheduler']: + if hasattr(self.scheduler, 'T_cur') and self.scheduler.T_cur == 0 and epoch != 1: + if self.use_swa: + self.swa_model.update_parameters(self.model) + if self.use_se: + model_copy = deepcopy(self.swa_model) if self.use_swa else deepcopy(self.model) + model_copy.cpu() + self.model_snapshots.append(model_copy) + self.model_snapshots = self.model_snapshots[-self.se_lastk:] + else: + if epoch > self._budget_threshold: + if self.use_swa: + self.swa_model.update_parameters(self.model) + if self.use_se: + model_copy = deepcopy(self.swa_model) if self.use_swa else deepcopy(self.model) + model_copy.cpu() + self.model_snapshots.append(model_copy) + self.model_snapshots = self.model_snapshots[-self.se_lastk:] return False def _scheduler_step( @@ -493,11 +541,23 @@ def criterion_preparation(self, y_a: torch.Tensor, y_b: torch.Tensor = None, lam @staticmethod def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, weighted_loss: Tuple[Tuple, bool] = ((True, False), True), - use_swa: Tuple[Tuple, bool] = ((True, False), False) + use_swa: Tuple[Tuple, bool] = ((True, False), True), + use_se: Tuple[Tuple, bool] = ((True, False), True), ) -> ConfigurationSpace: weighted_loss = CategoricalHyperparameter("weighted_loss", choices=weighted_loss[0], default_value=weighted_loss[1]) + use_swa = CategoricalHyperparameter("use_swa", choices=use_swa[0], default_value=use_swa[1]) + use_se = CategoricalHyperparameter("use_se", choices=use_se[0], default_value=use_se[1]) + + # Note, this is not easy to be considered as a hyperparameter. + # When used with cyclic learning rates, it depends on the number + # of restarts. + se_lastk = Constant('se_lastk', 3) + cs = ConfigurationSpace() + cs.add_hyperparameters([use_swa, use_se, se_lastk]) + cond = EqualsCondition(se_lastk, use_se, True) + cs.add_condition(cond) if dataset_properties is not None: if STRING_TO_TASK_TYPES[dataset_properties['task_type']] not in CLASSIFICATION_TASKS: cs.add_hyperparameters([weighted_loss]) diff --git a/autoPyTorch/pipeline/components/training/trainer/utils.py b/autoPyTorch/pipeline/components/training/trainer/utils.py new file mode 100644 index 000000000..135dd4584 --- /dev/null +++ b/autoPyTorch/pipeline/components/training/trainer/utils.py @@ -0,0 +1,27 @@ +import re +from typing import Dict + +import torch +from torch.nn.parameter import Parameter + + +def update_model_state_dict_from_swa(model: torch.nn.Module, swa_state_dict: Dict) -> None: + """ + swa model adds a module keyword to each parameter, + this function updates the state dict of the model + using the state dict of the swa model + Args: + model: + swa_state_dict: + + Returns: + + """ + model_state = model.state_dict() + for name, param in swa_state_dict.items(): + name = re.sub('module.', '', name) + if name not in model_state.keys(): + continue + # if isinstance(param, Parameter): + # param = param.data + model_state[name].copy_(param) \ No newline at end of file From b845d4d3c70d8e157e635e02227c3d50f97d7a68 Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Mon, 22 Feb 2021 13:40:15 +0100 Subject: [PATCH 03/50] ADD tests for SWA, SE Add pytest_mock to test dependencies lookahead in progress add lookahead hyperparameters --- .../setup/lr_scheduler/ReduceLROnPlateau.py | 3 +- .../components/setup/network/base_network.py | 5 +- .../training/trainer/StandardTrainer.py | 39 +---- .../components/training/trainer/__init__.py | 20 ++- .../training/trainer/base_trainer.py | 43 +++-- .../components/training/trainer/utils.py | 148 +++++++++++++++++- setup.py | 1 + .../test_tabular_classification.py | 28 +++- 8 files changed, 233 insertions(+), 54 deletions(-) diff --git a/autoPyTorch/pipeline/components/setup/lr_scheduler/ReduceLROnPlateau.py b/autoPyTorch/pipeline/components/setup/lr_scheduler/ReduceLROnPlateau.py index 4eda659d8..00503cb7e 100644 --- a/autoPyTorch/pipeline/components/setup/lr_scheduler/ReduceLROnPlateau.py +++ b/autoPyTorch/pipeline/components/setup/lr_scheduler/ReduceLROnPlateau.py @@ -1,5 +1,4 @@ -from typing import Any, Dict, Optional, Union - +from typing import Any, Dict, Optional, Tuple, Union from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( CategoricalHyperparameter, diff --git a/autoPyTorch/pipeline/components/setup/network/base_network.py b/autoPyTorch/pipeline/components/setup/network/base_network.py index 9970371a5..bd5f3a8b1 100644 --- a/autoPyTorch/pipeline/components/setup/network/base_network.py +++ b/autoPyTorch/pipeline/components/setup/network/base_network.py @@ -115,15 +115,16 @@ def predict(self, loader: torch.utils.data.DataLoader) -> torch.Tensor: """ if len(self.network_snapshots) == 0: assert self.network is not None - return self._predict(network=self.network, loader=loader).cpu().numpy() + return self._predict(network=self.network.float(), loader=loader).cpu().numpy() else: # if there are network snapshots, # take average of predictions of all snapshots Y_snapshot_preds = list() for network in self.network_snapshots: - Y_snapshot_preds.append(self._predict(network, loader)) + Y_snapshot_preds.append(self._predict(network.float(), loader)) Y_snapshot_preds = torch.stack(Y_snapshot_preds) + assert isinstance(Y_snapshot_preds, torch.Tensor) return Y_snapshot_preds.mean(dim=0).cpu().numpy() def _predict(self, network: torch.nn.Module, loader: torch.utils.data.DataLoader) -> torch.Tensor: diff --git a/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py b/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py index 3c0535a3a..103ac1969 100644 --- a/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py @@ -1,14 +1,8 @@ from typing import Callable, Dict, Optional, Tuple, Union from ConfigSpace.configuration_space import ConfigurationSpace -<<<<<<< HEAD -from ConfigSpace.hyperparameters import ( - CategoricalHyperparameter, -) -======= from ConfigSpace.hyperparameters import CategoricalHyperparameter ->>>>>>> swa working, se in progress import numpy as np import torch @@ -16,7 +10,6 @@ from autoPyTorch.constants import CLASSIFICATION_TASKS, STRING_TO_TASK_TYPES from autoPyTorch.datasets.base_dataset import BaseDatasetPropertiesType from autoPyTorch.pipeline.components.training.trainer.base_trainer import BaseTrainerComponent -from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter class StandardTrainer(BaseTrainerComponent): @@ -24,7 +17,9 @@ def __init__(self, weighted_loss: bool = False, use_swa: bool = False, use_se: bool = False, se_lastk: int = 3, - random_state: typing.Optional[np.random.RandomState] = None): + use_lookahead_optimizer: bool = True, + random_state: Optional[Union[np.random.RandomState, int]] = None, + **lookahead_config): """ This class handles the training of a network for a single given epoch. @@ -36,7 +31,9 @@ def __init__(self, weighted_loss: bool = False, weighted_loss=weighted_loss, use_swa=use_swa, use_se=use_se, - se_lastk=se_lastk) + se_lastk=se_lastk, + use_lookahead_optimizer=use_lookahead_optimizer, + **lookahead_config) def data_preparation(self, X: torch.Tensor, y: torch.Tensor, ) -> Tuple[torch.Tensor, Dict[str, np.ndarray]]: @@ -62,8 +59,8 @@ def criterion_preparation(self, y_a: torch.Tensor, y_b: torch.Tensor = None, lam return lambda criterion, pred: criterion(pred, y_a) @staticmethod - def get_properties(dataset_properties: typing.Optional[typing.Dict[str, typing.Any]] = None - ) -> typing.Dict[str, typing.Union[str, bool]]: + def get_properties(dataset_properties: Optional[Dict[str, Any]] = None + ) -> Dict[str, Union[str, bool]]: return { 'shortname': 'StandardTrainer', 'name': 'StandardTrainer', @@ -71,23 +68,3 @@ def get_properties(dataset_properties: typing.Optional[typing.Dict[str, typing.A 'handles_image': True, 'handles_time_series': True, } - - @staticmethod - def get_hyperparameter_search_space( - dataset_properties: Optional[Dict[str, BaseDatasetPropertiesType]] = None, - weighted_loss: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="weighted_loss", - value_range=(True, False), - default_value=True), - use_swa: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_swa", - value_range=(True, False), - default_value=True), - use_se: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_se", - value_range=(True, False), - default_value=True), - ) -> ConfigurationSpace: - cs = ConfigurationSpace() - if dataset_properties is not None: - if STRING_TO_TASK_TYPES[str(dataset_properties['task_type'])] in CLASSIFICATION_TASKS: - add_hyperparameter(cs, weighted_loss, CategoricalHyperparameter) - - return cs diff --git a/autoPyTorch/pipeline/components/training/trainer/__init__.py b/autoPyTorch/pipeline/components/training/trainer/__init__.py index f45f832c6..7d99d3017 100755 --- a/autoPyTorch/pipeline/components/training/trainer/__init__.py +++ b/autoPyTorch/pipeline/components/training/trainer/__init__.py @@ -28,13 +28,13 @@ ) from autoPyTorch.pipeline.components.training.losses import get_loss from autoPyTorch.pipeline.components.training.metrics.utils import get_metrics -from autoPyTorch.pipeline.components.training.trainer.utils import update_model_state_dict_from_swa from autoPyTorch.pipeline.components.training.trainer.base_trainer import ( BaseTrainerComponent, BudgetTracker, RunSummary, ) from autoPyTorch.utils.common import FitRequirement, get_device_from_fit_dictionary +from autoPyTorch.pipeline.components.training.trainer.utils import update_model_state_dict_from_swa from autoPyTorch.utils.logging_ import get_named_client_logger trainer_directory = os.path.split(__file__)[0] @@ -278,10 +278,22 @@ def fit(self, X: Dict[str, Any], y: Any = None, **kwargs: Any) -> autoPyTorchCom ) # Add snapshots to base network to enable # predicting with snapshot ensemble + self.choice = cast(autoPyTorchComponent, self.choice) if self.choice.use_se: X['network_snapshots'].extend(self.choice.model_snapshots) - return cast(autoPyTorchComponent, self.choice) + if X['use_pynisher']: + # Normally the X[network] is a pointer to the object, so at the + # end, when we train using X, the pipeline network is updated for free + # If we do multiprocessing (because of pynisher) we have to update + # X[network] manually. we do so in a way that every pipeline component + # can see this new network -- via an update, not overwrite of the pointer + state_dict = state_dict.result + X['network'].load_state_dict(state_dict) + + # TODO: when have the optimizer code, the pynisher object might have failed + # We should process this function as Failure if so trough fit_function.exit_status + return self.choice def prepare_trainer(self, X: Dict) -> None: """ @@ -433,12 +445,12 @@ def _fit(self, X: Dict[str, Any], y: Any = None, **kwargs: Any) -> 'TrainerChoic raise RuntimeError("Budget exhausted without finishing an epoch.") if self.choice.use_swa: # update batch norm statistics - swa_utils.update_bn(X['train_data_loader'], self.choice.swa_model) + swa_utils.update_bn(X['train_data_loader'], self.choice.swa_model.double()) # change model update_model_state_dict_from_swa(X['network'], self.choice.swa_model.state_dict()) if self.choice.use_se: for model in self.choice.model_snapshots: - swa_utils.update_bn(X['train_data_loader'], model) + swa_utils.update_bn(X['train_data_loader'], model.double()) # wrap up -- add score if not evaluating every epoch if not self.eval_valid_each_epoch(X): diff --git a/autoPyTorch/pipeline/components/training/trainer/base_trainer.py b/autoPyTorch/pipeline/components/training/trainer/base_trainer.py index b04e42bca..0c55558dc 100644 --- a/autoPyTorch/pipeline/components/training/trainer/base_trainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/base_trainer.py @@ -1,6 +1,5 @@ -from copy import deepcopy -from queue import LifoQueue import time +from copy import deepcopy from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union from ConfigSpace.conditions import EqualsCondition @@ -9,9 +8,6 @@ CategoricalHyperparameter, Constant ) - -from ConfigSpace.configuration_space import ConfigurationSpace -from ConfigSpace.hyperparameters import CategoricalHyperparameter import numpy as np import pandas as pd @@ -32,6 +28,7 @@ FORECASTING_METRICS, REGRESSION_METRICS, ) +from autoPyTorch.pipeline.components.training.trainer.utils import Lookahead from autoPyTorch.pipeline.components.training.metrics.utils import calculate_score from autoPyTorch.utils.common import FitRequirement from autoPyTorch.utils.implementations import get_loss_weight_strategy @@ -216,7 +213,9 @@ def __init__(self, weighted_loss: bool = False, use_swa: bool = False, use_se: bool = False, se_lastk: int = 3, - random_state: Optional[Union[np.random.RandomState, int]] = None) -> None: + use_lookahead_optimizer: bool = True, + random_state: Optional[Union[np.random.RandomState, int]] = None, + **lookahead_config) -> None: if random_state is None: # A trainer components need a random state for # sampling -- for example in MixUp training @@ -228,6 +227,8 @@ def __init__(self, weighted_loss: bool = False, self.use_swa = use_swa self.use_se = use_se self.se_lastk = se_lastk + self.use_lookahead_optimizer = use_lookahead_optimizer + self.lookahead_config = lookahead_config self.add_fit_requirements([ FitRequirement("is_cyclic_scheduler", (bool,), user_defined=False, dataset_property=False), ]) @@ -269,13 +270,19 @@ def prepare( self.swa_model = swa_utils.AveragedModel(self.model) # in case we are using se or swa, initialise budget_threshold to know when to start swa or se - self._budget_threshold: int = int(0.75 * budget_tracker.max_epochs) + self._budget_threshold = 0 + if self.use_swa or self.use_se: + assert budget_tracker.max_epochs is not None, "Can only use stochastic weight averaging or snapshot " \ + "ensemble when budget is epochs" + self._budget_threshold = int(0.75 * budget_tracker.max_epochs) # in case we are using se, initialise list to store model snapshots if self.use_se: self.model_snapshots: List[torch.nn.Module] = list() # setup the optimizers + if self.use_lookahead_optimizer: + optimizer = Lookahead(optimizer=optimizer, config=self.lookahead_config) self.optimizer = optimizer # The budget tracker @@ -543,6 +550,10 @@ def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, weighted_loss: Tuple[Tuple, bool] = ((True, False), True), use_swa: Tuple[Tuple, bool] = ((True, False), True), use_se: Tuple[Tuple, bool] = ((True, False), True), + se_lastk: Tuple[Tuple, int] = ((3,), 3), + use_lookahead_optimizer: Tuple[Tuple, bool] = ((True, False), True), + la_steps: Tuple[Tuple, int, bool] = ((5, 10), 6, False), + la_alpha: Tuple[Tuple, float, bool] = ((0.5, 0.8), 0.6, False), ) -> ConfigurationSpace: weighted_loss = CategoricalHyperparameter("weighted_loss", choices=weighted_loss[0], default_value=weighted_loss[1]) @@ -552,12 +563,26 @@ def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, # Note, this is not easy to be considered as a hyperparameter. # When used with cyclic learning rates, it depends on the number # of restarts. - se_lastk = Constant('se_lastk', 3) + se_lastk = Constant('se_lastk', se_lastk[1]) + + use_lookahead_optimizer = CategoricalHyperparameter("use_lookahead_optimizer", + choices=use_lookahead_optimizer[0], + default_value=use_lookahead_optimizer[1]) + + config_space = Lookahead.get_hyperparameter_search_space(la_steps=la_steps, + la_alpha=la_alpha) + parent_hyperparameter = {'parent': use_lookahead_optimizer, 'value': True} cs = ConfigurationSpace() - cs.add_hyperparameters([use_swa, use_se, se_lastk]) + cs.add_hyperparameters([use_swa, use_se, se_lastk, use_lookahead_optimizer]) + cs.add_configuration_space( + 'lookahead_optimizer', + config_space, + parent_hyperparameter=parent_hyperparameter + ) cond = EqualsCondition(se_lastk, use_se, True) cs.add_condition(cond) + if dataset_properties is not None: if STRING_TO_TASK_TYPES[dataset_properties['task_type']] not in CLASSIFICATION_TASKS: cs.add_hyperparameters([weighted_loss]) diff --git a/autoPyTorch/pipeline/components/training/trainer/utils.py b/autoPyTorch/pipeline/components/training/trainer/utils.py index 135dd4584..f5e22a633 100644 --- a/autoPyTorch/pipeline/components/training/trainer/utils.py +++ b/autoPyTorch/pipeline/components/training/trainer/utils.py @@ -1,8 +1,15 @@ +from collections import defaultdict import re -from typing import Dict +from typing import Dict, Tuple + +from ConfigSpace.configuration_space import ConfigurationSpace +from ConfigSpace.hyperparameters import ( + UniformFloatHyperparameter, + UniformIntegerHyperparameter +) import torch -from torch.nn.parameter import Parameter +from torch.optim.optimizer import Optimizer def update_model_state_dict_from_swa(model: torch.nn.Module, swa_state_dict: Dict) -> None: @@ -22,6 +29,137 @@ def update_model_state_dict_from_swa(model: torch.nn.Module, swa_state_dict: Dic name = re.sub('module.', '', name) if name not in model_state.keys(): continue - # if isinstance(param, Parameter): - # param = param.data - model_state[name].copy_(param) \ No newline at end of file + model_state[name].copy_(param) + + +class Lookahead(Optimizer): + r"""PyTorch implementation of the lookahead wrapper. + Lookahead Optimizer: https://arxiv.org/abs/1907.08610 + """ + + def __init__(self, optimizer, config): + """optimizer: inner optimizer + la_steps (int): number of lookahead steps + la_alpha (float): linear interpolation factor. 1.0 recovers the inner optimizer. + pullback_momentum (str): change to inner optimizer momentum on interpolation update + """ + self.optimizer = optimizer + self._la_step = 0 # counter for inner optimizer + self.la_alpha = config["lookahead_optimizer:la_alpha"] + self.la_alpha = torch.tensor(self.la_alpha) + self._total_la_steps = config["lookahead_optimizer:la_steps"] + # TODO possibly incorporate different momentum options when using SGD + pullback_momentum = "none" + pullback_momentum = pullback_momentum.lower() + assert pullback_momentum in ["reset", "pullback", "none"] + self.pullback_momentum = pullback_momentum + + self.state = defaultdict(dict) + + # Cache the current optimizer parameters + for group in optimizer.param_groups: + for p in group['params']: + param_state = self.state[p] + param_state['cached_params'] = torch.zeros_like(p.data) + param_state['cached_params'].copy_(p.data) + if self.pullback_momentum == "pullback": + param_state['cached_mom'] = torch.zeros_like(p.data) + + def __getstate__(self): + return { + 'state': self.state, + 'optimizer': self.optimizer, + 'la_alpha': self.la_alpha, + '_la_step': self._la_step, + '_total_la_steps': self._total_la_steps, + 'pullback_momentum': self.pullback_momentum + } + + def zero_grad(self): + self.optimizer.zero_grad() + + def get_la_step(self): + return self._la_step + + def state_dict(self): + return self.optimizer.state_dict() + + def load_state_dict(self, state_dict): + self.optimizer.load_state_dict(state_dict) + + def _backup_and_load_cache(self): + """Useful for performing evaluation on the slow weights (which typically generalize better) + """ + for group in self.optimizer.param_groups: + for p in group['params']: + param_state = self.state[p] + param_state['backup_params'] = torch.zeros_like(p.data) + param_state['backup_params'].copy_(p.data) + p.data.copy_(param_state['cached_params']) + + def _clear_and_load_backup(self): + for group in self.optimizer.param_groups: + for p in group['params']: + param_state = self.state[p] + p.data.copy_(param_state['backup_params']) + del param_state['backup_params'] + + @property + def param_groups(self): + return self.optimizer.param_groups + + def step(self, closure=None): + """Performs a single Lookahead optimization step. + Arguments: + closure (callable, optional): A closure that reevaluates the model + and returns the loss. + """ + loss = self.optimizer.step(closure) + self._la_step += 1 + + if self._la_step >= self._total_la_steps: + self._la_step = 0 + # Lookahead and cache the current optimizer parameters + for group in self.optimizer.param_groups: + for p in group['params']: + param_state = self.state[p] + p.data.mul_(self.la_alpha).add_(1.0 - self.la_alpha, param_state['cached_params']) # crucial line + param_state['cached_params'].copy_(p.data) + if self.pullback_momentum == "pullback": + internal_momentum = self.optimizer.state[p]["momentum_buffer"] + self.optimizer.state[p]["momentum_buffer"] = internal_momentum.mul_(self.la_alpha).add_( + 1.0 - self.la_alpha, param_state["cached_mom"]) + param_state["cached_mom"] = self.optimizer.state[p]["momentum_buffer"] + elif self.pullback_momentum == "reset": + self.optimizer.state[p]["momentum_buffer"] = torch.zeros_like(p.data) + + return loss + + def to(self, device): + + self.la_alpha.to(device) + for group in self.optimizer.param_groups: + for p in group['params']: + param_state = self.state[p] + param_state['cached_params'] = param_state['cached_params'].to(device) + param_state['cached_params'].copy_(p.data) + if self.pullback_momentum == "pullback": + param_state['cached_mom'] = param_state['cached_mom'].to(device) + + @staticmethod + def get_hyperparameter_search_space( + la_steps: Tuple[Tuple, int, bool] = ((5, 10), 6, False), + la_alpha: Tuple[Tuple, float, bool] = ((0.5, 0.8), 0.6, False), + ): + cs = ConfigurationSpace() + la_steps = UniformIntegerHyperparameter('la_steps', lower=la_steps[0][0], + upper=la_steps[0][1], + default_value=la_steps[1], + log=la_steps[2]) + la_alpha = UniformFloatHyperparameter('la_alpha', lower=la_alpha[0][0], + upper=la_alpha[0][1], + default_value=la_alpha[1], + log=la_alpha[2]) + cs.add_hyperparameters([la_steps, la_alpha]) + + return cs diff --git a/setup.py b/setup.py index bd524276d..41f9f38f9 100755 --- a/setup.py +++ b/setup.py @@ -64,6 +64,7 @@ "pytest-cov", 'pytest-forked', 'pytest-subtests', + "pytest-mock" "codecov", "pep8", "mypy", diff --git a/test/test_pipeline/test_tabular_classification.py b/test/test_pipeline/test_tabular_classification.py index c679b931d..785f53032 100644 --- a/test/test_pipeline/test_tabular_classification.py +++ b/test/test_pipeline/test_tabular_classification.py @@ -15,6 +15,8 @@ import pytest +from pytest_mock import mocker # noqa F401 + import torch from torch.optim.lr_scheduler import _LRScheduler @@ -339,7 +341,7 @@ def test_set_range_search_space_updates(self, fit_dictionary_tabular): except AssertionError as e: # As we are setting num_layers to 1 for fully connected # head, units_layer does not exist in the configspace - assert 'fully_connected:units_layer' in e.args[0], e.args[0] + assert 'fully_connected:units_layer' in e.args[0] def test_set_choices_updates(self, fit_dictionary_tabular): dataset_properties = {'numerical_columns': [1], 'categorical_columns': [2], @@ -367,6 +369,30 @@ def test_set_choices_updates(self, fit_dictionary_tabular): search_space_updates=updates) self._assert_pipeline_search_space(pipeline, updates) + def test_swa_se(self, fit_dictionary, mocker): # noqa F811 + fit_dictionary['epochs'] = 10 + pipeline = TabularClassificationPipeline( + dataset_properties=fit_dictionary['dataset_properties'], + include={'lr_scheduler': ['CosineAnnealingWarmRestarts']}) + cs = pipeline.get_hyperparameter_search_space() + config = cs.get_default_configuration() + pipeline.set_hyperparameters(config) + + pipeline.fit(fit_dictionary.copy()) + X = pipeline.transform(fit_dictionary.copy()) + assert 'is_cyclic_scheduler' in X and X['is_cyclic_scheduler'] + + trainer = config.get('trainer:__choice__') + assert 'network_snapshots' in X and \ + len(X['network_snapshots']) == config.get(f'trainer:{trainer}:se_lastk') + + mocker.patch("autoPyTorch.pipeline.components.setup.network.base_network.NetworkComponent._predict", + return_value=torch.Tensor([1])) + # Assert that predict gives no error when swa and se are on + assert isinstance(pipeline.predict(fit_dictionary['X_train']), np.ndarray) + # As SE is True, _predict should be called 3 times + assert pipeline.named_steps['network']._predict.call_count == 3 + @pytest.mark.parametrize("fit_dictionary_tabular", ['iris'], indirect=True) def test_constant_pipeline_iris(fit_dictionary_tabular): From 8a2364d142228eee176f1c077c6745fae63672ed Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Mon, 22 Feb 2021 16:19:25 +0100 Subject: [PATCH 04/50] ADD tests for LookAhead Fix flake mypy Fix error in test --- .../training/trainer/StandardTrainer.py | 2 +- .../components/training/trainer/__init__.py | 23 ++++++++++++- .../training/trainer/base_trainer.py | 5 +-- .../components/training/trainer/utils.py | 34 +++++++++---------- .../test_tabular_classification.py | 6 +++- 5 files changed, 48 insertions(+), 22 deletions(-) diff --git a/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py b/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py index 103ac1969..0f635431f 100644 --- a/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py @@ -19,7 +19,7 @@ def __init__(self, weighted_loss: bool = False, se_lastk: int = 3, use_lookahead_optimizer: bool = True, random_state: Optional[Union[np.random.RandomState, int]] = None, - **lookahead_config): + **lookahead_config: Dict[str, Any]): """ This class handles the training of a network for a single given epoch. diff --git a/autoPyTorch/pipeline/components/training/trainer/__init__.py b/autoPyTorch/pipeline/components/training/trainer/__init__.py index 7d99d3017..c1e0c9581 100755 --- a/autoPyTorch/pipeline/components/training/trainer/__init__.py +++ b/autoPyTorch/pipeline/components/training/trainer/__init__.py @@ -33,8 +33,8 @@ BudgetTracker, RunSummary, ) +from autoPyTorch.pipeline.components.training.trainer.utils import Lookahead, update_model_state_dict_from_swa from autoPyTorch.utils.common import FitRequirement, get_device_from_fit_dictionary -from autoPyTorch.pipeline.components.training.trainer.utils import update_model_state_dict_from_swa from autoPyTorch.utils.logging_ import get_named_client_logger trainer_directory = os.path.split(__file__)[0] @@ -672,3 +672,24 @@ def __str__(self) -> str: """ Allow a nice understanding of what components where used """ string = str(self.run_summary) return string + + def _get_search_space_updates(self, prefix: Optional[str] = None) -> Dict[str, Tuple]: + """Get the search space updates with the given prefix + + Keyword Arguments: + prefix {str} -- Only return search space updates with given prefix (default: {None}) + + Returns: + dict -- Mapping of search space updates. Keys don't contain the prefix. + """ + updates = super()._get_search_space_updates(prefix=prefix) + + result: Dict[str, Tuple] = dict() + + # iterate over all search space updates of this node and filter the ones out, that have the given prefix + for key in updates.keys(): + if key.startswith(Lookahead.__name__): + result[key[len(Lookahead.__name__) + 1:]] = updates[key] + else: + result[key] = updates[key] + return result diff --git a/autoPyTorch/pipeline/components/training/trainer/base_trainer.py b/autoPyTorch/pipeline/components/training/trainer/base_trainer.py index 0c55558dc..854466106 100644 --- a/autoPyTorch/pipeline/components/training/trainer/base_trainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/base_trainer.py @@ -30,6 +30,7 @@ ) from autoPyTorch.pipeline.components.training.trainer.utils import Lookahead from autoPyTorch.pipeline.components.training.metrics.utils import calculate_score +from autoPyTorch.pipeline.components.training.trainer.utils import Lookahead from autoPyTorch.utils.common import FitRequirement from autoPyTorch.utils.implementations import get_loss_weight_strategy @@ -215,7 +216,7 @@ def __init__(self, weighted_loss: bool = False, se_lastk: int = 3, use_lookahead_optimizer: bool = True, random_state: Optional[Union[np.random.RandomState, int]] = None, - **lookahead_config) -> None: + **lookahead_config: Dict[str, Any]) -> None: if random_state is None: # A trainer components need a random state for # sampling -- for example in MixUp training @@ -576,7 +577,7 @@ def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, cs = ConfigurationSpace() cs.add_hyperparameters([use_swa, use_se, se_lastk, use_lookahead_optimizer]) cs.add_configuration_space( - 'lookahead_optimizer', + Lookahead.__name__, config_space, parent_hyperparameter=parent_hyperparameter ) diff --git a/autoPyTorch/pipeline/components/training/trainer/utils.py b/autoPyTorch/pipeline/components/training/trainer/utils.py index f5e22a633..10c2aeb9c 100644 --- a/autoPyTorch/pipeline/components/training/trainer/utils.py +++ b/autoPyTorch/pipeline/components/training/trainer/utils.py @@ -1,6 +1,6 @@ -from collections import defaultdict import re -from typing import Dict, Tuple +from collections import defaultdict +from typing import Any, Callable, Dict, List, Optional, Tuple from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( @@ -37,7 +37,7 @@ class Lookahead(Optimizer): Lookahead Optimizer: https://arxiv.org/abs/1907.08610 """ - def __init__(self, optimizer, config): + def __init__(self, optimizer: Optimizer, config: Dict[str, Any]) -> None: """optimizer: inner optimizer la_steps (int): number of lookahead steps la_alpha (float): linear interpolation factor. 1.0 recovers the inner optimizer. @@ -45,16 +45,16 @@ def __init__(self, optimizer, config): """ self.optimizer = optimizer self._la_step = 0 # counter for inner optimizer - self.la_alpha = config["lookahead_optimizer:la_alpha"] + self.la_alpha = config[f"{self.__class__.__name__}:la_alpha"] self.la_alpha = torch.tensor(self.la_alpha) - self._total_la_steps = config["lookahead_optimizer:la_steps"] + self._total_la_steps = config[f"{self.__class__.__name__}:la_steps"] # TODO possibly incorporate different momentum options when using SGD pullback_momentum = "none" pullback_momentum = pullback_momentum.lower() assert pullback_momentum in ["reset", "pullback", "none"] self.pullback_momentum = pullback_momentum - self.state = defaultdict(dict) + self.state: defaultdict = defaultdict(dict) # Cache the current optimizer parameters for group in optimizer.param_groups: @@ -65,7 +65,7 @@ def __init__(self, optimizer, config): if self.pullback_momentum == "pullback": param_state['cached_mom'] = torch.zeros_like(p.data) - def __getstate__(self): + def __getstate__(self) -> Dict[str, Any]: return { 'state': self.state, 'optimizer': self.optimizer, @@ -75,19 +75,19 @@ def __getstate__(self): 'pullback_momentum': self.pullback_momentum } - def zero_grad(self): + def zero_grad(self) -> None: self.optimizer.zero_grad() - def get_la_step(self): + def get_la_step(self) -> int: return self._la_step - def state_dict(self): + def state_dict(self) -> Dict[str, Any]: return self.optimizer.state_dict() - def load_state_dict(self, state_dict): + def load_state_dict(self, state_dict: Dict[str, Any]) -> None: self.optimizer.load_state_dict(state_dict) - def _backup_and_load_cache(self): + def _backup_and_load_cache(self) -> None: """Useful for performing evaluation on the slow weights (which typically generalize better) """ for group in self.optimizer.param_groups: @@ -97,7 +97,7 @@ def _backup_and_load_cache(self): param_state['backup_params'].copy_(p.data) p.data.copy_(param_state['cached_params']) - def _clear_and_load_backup(self): + def _clear_and_load_backup(self) -> None: for group in self.optimizer.param_groups: for p in group['params']: param_state = self.state[p] @@ -105,10 +105,10 @@ def _clear_and_load_backup(self): del param_state['backup_params'] @property - def param_groups(self): + def param_groups(self) -> List[Dict]: return self.optimizer.param_groups - def step(self, closure=None): + def step(self, closure: Optional[Callable] = None) -> torch.Tensor: """Performs a single Lookahead optimization step. Arguments: closure (callable, optional): A closure that reevaluates the model @@ -135,7 +135,7 @@ def step(self, closure=None): return loss - def to(self, device): + def to(self, device: str) -> None: self.la_alpha.to(device) for group in self.optimizer.param_groups: @@ -150,7 +150,7 @@ def to(self, device): def get_hyperparameter_search_space( la_steps: Tuple[Tuple, int, bool] = ((5, 10), 6, False), la_alpha: Tuple[Tuple, float, bool] = ((0.5, 0.8), 0.6, False), - ): + ) -> ConfigurationSpace: cs = ConfigurationSpace() la_steps = UniformIntegerHyperparameter('la_steps', lower=la_steps[0][0], upper=la_steps[0][1], diff --git a/test/test_pipeline/test_tabular_classification.py b/test/test_pipeline/test_tabular_classification.py index 785f53032..dc2ec6fcd 100644 --- a/test/test_pipeline/test_tabular_classification.py +++ b/test/test_pipeline/test_tabular_classification.py @@ -22,6 +22,7 @@ from autoPyTorch.pipeline.components.setup.early_preprocessor.utils import get_preprocess_transforms from autoPyTorch.pipeline.components.setup.lr_scheduler.NoScheduler import NoScheduler +from autoPyTorch.pipeline.components.training.trainer.utils import Lookahead from autoPyTorch.pipeline.tabular_classification import TabularClassificationPipeline from autoPyTorch.utils.common import FitRequirement from autoPyTorch.utils.hyperparameter_search_space_update import HyperparameterSearchSpaceUpdates, \ @@ -343,6 +344,7 @@ def test_set_range_search_space_updates(self, fit_dictionary_tabular): # head, units_layer does not exist in the configspace assert 'fully_connected:units_layer' in e.args[0] + def test_set_choices_updates(self, fit_dictionary_tabular): dataset_properties = {'numerical_columns': [1], 'categorical_columns': [2], 'task_type': 'tabular_classification', 'issparse': False, @@ -369,7 +371,8 @@ def test_set_choices_updates(self, fit_dictionary_tabular): search_space_updates=updates) self._assert_pipeline_search_space(pipeline, updates) - def test_swa_se(self, fit_dictionary, mocker): # noqa F811 + + def test_trainer_cocktails(self, fit_dictionary, mocker): # noqa F811 fit_dictionary['epochs'] = 10 pipeline = TabularClassificationPipeline( dataset_properties=fit_dictionary['dataset_properties'], @@ -392,6 +395,7 @@ def test_swa_se(self, fit_dictionary, mocker): # noqa F811 assert isinstance(pipeline.predict(fit_dictionary['X_train']), np.ndarray) # As SE is True, _predict should be called 3 times assert pipeline.named_steps['network']._predict.call_count == 3 + assert isinstance(pipeline.named_steps['trainer'].choice.optimizer, Lookahead) @pytest.mark.parametrize("fit_dictionary_tabular", ['iris'], indirect=True) From e2ee40a2a5c758d0fd8be3eb0a6491b7f82efdb6 Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Mon, 22 Feb 2021 18:12:03 +0100 Subject: [PATCH 05/50] Added default for lookahead config Addressed comments from francisco fixed issues after merge Add displayable path to model file assertion Add test for non cyclic scheduler --- .../components/setup/network/base_network.py | 21 ++--- .../training/trainer/MixUpTrainer.py | 1 - .../training/trainer/StandardTrainer.py | 10 +-- .../components/training/trainer/__init__.py | 7 +- .../training/trainer/base_trainer.py | 62 ++++++++++----- .../training/trainer/cutout_utils.py | 76 ++++++++++++++++-- .../training/trainer/mixup_utils.py | 79 ++++++++++++++++--- .../components/training/trainer/utils.py | 27 ++++--- .../components/training/test_training.py | 57 ++----------- .../test_tabular_classification.py | 31 ++++++-- test/utils.py | 71 +++++++++++++++++ 11 files changed, 320 insertions(+), 122 deletions(-) create mode 100644 test/utils.py diff --git a/autoPyTorch/pipeline/components/setup/network/base_network.py b/autoPyTorch/pipeline/components/setup/network/base_network.py index bd5f3a8b1..daba6307d 100644 --- a/autoPyTorch/pipeline/components/setup/network/base_network.py +++ b/autoPyTorch/pipeline/components/setup/network/base_network.py @@ -21,14 +21,15 @@ class NetworkComponent(autoPyTorchTrainingComponent): """ def __init__( - self, - network: Optional[torch.nn.Module] = None, - random_state: Optional[np.random.RandomState] = None + self, + network: Optional[torch.nn.Module] = None, + network_snapshots: Optional[List[torch.nn.Module]] = None, + random_state: Optional[np.random.RandomState] = None, ) -> None: super(NetworkComponent, self).__init__() self.network = network - self.network_snapshots: List[torch.nn.Module] = [] + self.network_snapshots = network_snapshots if network_snapshots is not None else [] self.random_state = random_state self.device = None self.add_fit_requirements([ @@ -115,19 +116,19 @@ def predict(self, loader: torch.utils.data.DataLoader) -> torch.Tensor: """ if len(self.network_snapshots) == 0: assert self.network is not None - return self._predict(network=self.network.float(), loader=loader).cpu().numpy() + return self._predict(network=self.network, loader=loader).cpu().numpy() else: # if there are network snapshots, # take average of predictions of all snapshots - Y_snapshot_preds = list() + Y_snapshot_preds: List[torch.Tensor] = list() for network in self.network_snapshots: - Y_snapshot_preds.append(self._predict(network.float(), loader)) - Y_snapshot_preds = torch.stack(Y_snapshot_preds) - assert isinstance(Y_snapshot_preds, torch.Tensor) - return Y_snapshot_preds.mean(dim=0).cpu().numpy() + Y_snapshot_preds.append(self._predict(network, loader)) + Y_snapshot_preds_tensor = torch.stack(Y_snapshot_preds) + return Y_snapshot_preds_tensor.mean(dim=0).cpu().numpy() def _predict(self, network: torch.nn.Module, loader: torch.utils.data.DataLoader) -> torch.Tensor: + network.float() network.eval() # Batch prediction Y_batch_preds = list() diff --git a/autoPyTorch/pipeline/components/training/trainer/MixUpTrainer.py b/autoPyTorch/pipeline/components/training/trainer/MixUpTrainer.py index 7cf443126..be909abfb 100644 --- a/autoPyTorch/pipeline/components/training/trainer/MixUpTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/MixUpTrainer.py @@ -22,7 +22,6 @@ def data_preparation(self, X: np.ndarray, y: np.ndarray, on a different way. That is, in standard training we provide the data to the network as we receive it to the loader. Some regularization techniques, like mixup alter the data. - Args: X (torch.Tensor): The batch training features y (torch.Tensor): The batch training labels diff --git a/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py b/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py index 0f635431f..b3c47eaee 100644 --- a/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py @@ -14,8 +14,8 @@ class StandardTrainer(BaseTrainerComponent): def __init__(self, weighted_loss: bool = False, - use_swa: bool = False, - use_se: bool = False, + use_stochastic_weight_averaging: bool = False, + use_snapshot_ensemble: bool = False, se_lastk: int = 3, use_lookahead_optimizer: bool = True, random_state: Optional[Union[np.random.RandomState, int]] = None, @@ -29,8 +29,8 @@ def __init__(self, weighted_loss: bool = False, """ super().__init__(random_state=random_state, weighted_loss=weighted_loss, - use_swa=use_swa, - use_se=use_se, + use_stochastic_weight_averaging=use_stochastic_weight_averaging, + use_snapshot_ensemble=use_snapshot_ensemble, se_lastk=se_lastk, use_lookahead_optimizer=use_lookahead_optimizer, **lookahead_config) @@ -60,7 +60,7 @@ def criterion_preparation(self, y_a: torch.Tensor, y_b: torch.Tensor = None, lam @staticmethod def get_properties(dataset_properties: Optional[Dict[str, Any]] = None - ) -> Dict[str, Union[str, bool]]: + ) -> Dict[str, Union[str, bool]]: return { 'shortname': 'StandardTrainer', 'name': 'StandardTrainer', diff --git a/autoPyTorch/pipeline/components/training/trainer/__init__.py b/autoPyTorch/pipeline/components/training/trainer/__init__.py index c1e0c9581..dc0c8477a 100755 --- a/autoPyTorch/pipeline/components/training/trainer/__init__.py +++ b/autoPyTorch/pipeline/components/training/trainer/__init__.py @@ -279,7 +279,7 @@ def fit(self, X: Dict[str, Any], y: Any = None, **kwargs: Any) -> autoPyTorchCom # Add snapshots to base network to enable # predicting with snapshot ensemble self.choice = cast(autoPyTorchComponent, self.choice) - if self.choice.use_se: + if self.choice.use_snapshot_ensemble: X['network_snapshots'].extend(self.choice.model_snapshots) if X['use_pynisher']: @@ -443,12 +443,13 @@ def _fit(self, X: Dict[str, Any], y: Any = None, **kwargs: Any) -> 'TrainerChoic if self.run_summary.is_empty(): raise RuntimeError("Budget exhausted without finishing an epoch.") - if self.choice.use_swa: + + if self.choice.use_stochastic_weight_averaging: # update batch norm statistics swa_utils.update_bn(X['train_data_loader'], self.choice.swa_model.double()) # change model update_model_state_dict_from_swa(X['network'], self.choice.swa_model.state_dict()) - if self.choice.use_se: + if self.choice.use_snapshot_ensemble: for model in self.choice.model_snapshots: swa_utils.update_bn(X['train_data_loader'], model.double()) diff --git a/autoPyTorch/pipeline/components/training/trainer/base_trainer.py b/autoPyTorch/pipeline/components/training/trainer/base_trainer.py index 854466106..a79d3a74b 100644 --- a/autoPyTorch/pipeline/components/training/trainer/base_trainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/base_trainer.py @@ -209,10 +209,24 @@ def is_empty(self) -> bool: class BaseTrainerComponent(autoPyTorchTrainingComponent): - - def __init__(self, weighted_loss: bool = False, - use_swa: bool = False, - use_se: bool = False, + """ + Base class for training + Args: + weighted_loss (bool, default=True): In case for classification, whether to weight + the loss function according to the distribution of classes in the target + use_stochastic_weight_averaging (bool, default=True): whether to use stochastic + weight averaging + use_snapshot_ensemble (bool, default=True): whether to use snapshot + ensemble + se_lastk (int, default=3): Number of snapshots of the network to maintain + use_lookahead_optimizer (bool, default=True): whether to use lookahead + optimizer + random_state: + **lookahead_config: + """ + def __init__(self, weighted_loss: bool = True, + use_stochastic_weight_averaging: bool = True, + use_snapshot_ensemble: bool = True, se_lastk: int = 3, use_lookahead_optimizer: bool = True, random_state: Optional[Union[np.random.RandomState, int]] = None, @@ -225,10 +239,14 @@ def __init__(self, weighted_loss: bool = False, self.random_state = random_state super().__init__(random_state=self.random_state) self.weighted_loss = weighted_loss - self.use_swa = use_swa - self.use_se = use_se + self.use_stochastic_weight_averaging = use_stochastic_weight_averaging + self.use_snapshot_ensemble = use_snapshot_ensemble self.se_lastk = se_lastk self.use_lookahead_optimizer = use_lookahead_optimizer + # Add default values for the lookahead optimizer + if len(lookahead_config) == 0: + lookahead_config = {f'{Lookahead.__name__}:la_steps': 6, + f'{Lookahead.__name__}:la_alpha': 0.6} self.lookahead_config = lookahead_config self.add_fit_requirements([ FitRequirement("is_cyclic_scheduler", (bool,), user_defined=False, dataset_property=False), @@ -267,18 +285,18 @@ def prepare( self.model = model.to(device) # in case we are using swa, maintain an averaged model, - if self.use_swa: + if self.use_stochastic_weight_averaging: self.swa_model = swa_utils.AveragedModel(self.model) # in case we are using se or swa, initialise budget_threshold to know when to start swa or se self._budget_threshold = 0 - if self.use_swa or self.use_se: + if self.use_stochastic_weight_averaging or self.use_snapshot_ensemble: assert budget_tracker.max_epochs is not None, "Can only use stochastic weight averaging or snapshot " \ "ensemble when budget is epochs" self._budget_threshold = int(0.75 * budget_tracker.max_epochs) # in case we are using se, initialise list to store model snapshots - if self.use_se: + if self.use_snapshot_ensemble: self.model_snapshots: List[torch.nn.Module] = list() # setup the optimizers @@ -316,19 +334,21 @@ def on_epoch_end(self, X: Dict[str, Any], epoch: int) -> bool: """ if X['is_cyclic_scheduler']: if hasattr(self.scheduler, 'T_cur') and self.scheduler.T_cur == 0 and epoch != 1: - if self.use_swa: + if self.use_stochastic_weight_averaging: self.swa_model.update_parameters(self.model) - if self.use_se: - model_copy = deepcopy(self.swa_model) if self.use_swa else deepcopy(self.model) + if self.use_snapshot_ensemble: + model_copy = deepcopy(self.swa_model) if self.use_stochastic_weight_averaging \ + else deepcopy(self.model) model_copy.cpu() self.model_snapshots.append(model_copy) self.model_snapshots = self.model_snapshots[-self.se_lastk:] else: if epoch > self._budget_threshold: - if self.use_swa: + if self.use_stochastic_weight_averaging: self.swa_model.update_parameters(self.model) - if self.use_se: - model_copy = deepcopy(self.swa_model) if self.use_swa else deepcopy(self.model) + if self.use_snapshot_ensemble: + model_copy = deepcopy(self.swa_model) if self.use_stochastic_weight_averaging \ + else deepcopy(self.model) model_copy.cpu() self.model_snapshots.append(model_copy) self.model_snapshots = self.model_snapshots[-self.se_lastk:] @@ -549,8 +569,8 @@ def criterion_preparation(self, y_a: torch.Tensor, y_b: torch.Tensor = None, lam @staticmethod def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, weighted_loss: Tuple[Tuple, bool] = ((True, False), True), - use_swa: Tuple[Tuple, bool] = ((True, False), True), - use_se: Tuple[Tuple, bool] = ((True, False), True), + use_stochastic_weight_averaging: Tuple[Tuple, bool] = ((True, False), True), + use_snapshot_ensemble: Tuple[Tuple, bool] = ((True, False), True), se_lastk: Tuple[Tuple, int] = ((3,), 3), use_lookahead_optimizer: Tuple[Tuple, bool] = ((True, False), True), la_steps: Tuple[Tuple, int, bool] = ((5, 10), 6, False), @@ -558,8 +578,12 @@ def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, ) -> ConfigurationSpace: weighted_loss = CategoricalHyperparameter("weighted_loss", choices=weighted_loss[0], default_value=weighted_loss[1]) - use_swa = CategoricalHyperparameter("use_swa", choices=use_swa[0], default_value=use_swa[1]) - use_se = CategoricalHyperparameter("use_se", choices=use_se[0], default_value=use_se[1]) + use_swa = CategoricalHyperparameter("use_stochastic_weight_averaging", + choices=use_stochastic_weight_averaging[0], + default_value=use_stochastic_weight_averaging[1]) + use_se = CategoricalHyperparameter("use_snapshot_ensemble", + choices=use_snapshot_ensemble[0], + default_value=use_snapshot_ensemble[1]) # Note, this is not easy to be considered as a hyperparameter. # When used with cyclic learning rates, it depends on the number diff --git a/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py b/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py index 09d3c653b..cc3b03018 100644 --- a/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py +++ b/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py @@ -1,22 +1,30 @@ -import typing +from typing import Any, Callable, Dict, Optional, Tuple +from ConfigSpace.conditions import EqualsCondition from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( CategoricalHyperparameter, + Constant, UniformFloatHyperparameter, ) import numpy as np from autoPyTorch.constants import CLASSIFICATION_TASKS, STRING_TO_TASK_TYPES -from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter +from autoPyTorch.pipeline.components.training.trainer.utils import Lookahead +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter class CutOut: def __init__(self, patch_ratio: float, cutout_prob: float, weighted_loss: bool = False, - random_state: typing.Optional[np.random.RandomState] = None): + random_state: Optional[np.random.RandomState] = None, + use_stochastic_weight_averaging: bool = False, + use_snapshot_ensemble: bool = False, + se_lastk: int = 3, + use_lookahead_optimizer: bool = True, + **lookahead_config: Any): """ This class handles the training of a network for a single given epoch. @@ -25,36 +33,88 @@ def __init__(self, patch_ratio: float, cutout_prob (float): The probability of occurrence of this regulatization """ + self.use_stochastic_weight_averaging = use_stochastic_weight_averaging self.weighted_loss = weighted_loss + self.random_state = random_state + self.use_snapshot_ensemble = use_snapshot_ensemble + self.se_lastk = se_lastk + self.use_lookahead_optimizer = use_lookahead_optimizer + # Add default values for the lookahead optimizer + if len(lookahead_config) == 0: + lookahead_config = {f'{Lookahead.__name__}:la_steps': 6, + f'{Lookahead.__name__}:la_alpha': 0.6} + self.lookahead_config = lookahead_config self.patch_ratio = patch_ratio self.cutout_prob = cutout_prob - self.random_state = random_state def criterion_preparation(self, y_a: np.ndarray, y_b: np.ndarray = None, lam: float = 1.0 - ) -> typing.Callable: + ) -> Callable: return lambda criterion, pred: lam * criterion(pred, y_a) + (1 - lam) * criterion(pred, y_b) @staticmethod def get_hyperparameter_search_space( - dataset_properties: typing.Optional[typing.Dict] = None, + dataset_properties: Optional[Dict] = None, weighted_loss: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="weighted_loss", value_range=(True, False), default_value=True), patch_ratio: HyperparameterSearchSpace = HyperparameterSearchSpace( - hyperparameter="alpha", + hyperparameter="patch_ratio", value_range=(0, 1), default_value=0.2), cutout_prob: HyperparameterSearchSpace = HyperparameterSearchSpace( - hyperparameter="alpha", + hyperparameter="cutout_prob", value_range=(0, 1), default_value=0.2), + la_steps: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="la_steps", + value_range=(5, 10), + default_value=6, + log=False), + la_alpha: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="la_alpha", + value_range=(0.5, 0.8), + default_value=0.6, + log=False), + use_lookahead_optimizer: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="use_lookahead_optimizer", + value_range=(True, False), + default_value=True), + use_stochastic_weight_averaging: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="use_stochastic_weight_averaging", + value_range=(True, False), + default_value=True), + use_snapshot_ensemble: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="use_snapshot_ensemble", + value_range=(True, False), + default_value=True), + se_lastk: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="se_lastk", + value_range=(3, ), + default_value=3), ) -> ConfigurationSpace: cs = ConfigurationSpace() add_hyperparameter(cs, patch_ratio, UniformFloatHyperparameter) add_hyperparameter(cs, cutout_prob, UniformFloatHyperparameter) + add_hyperparameter(cs, se_lastk, Constant) + add_hyperparameter(cs, use_stochastic_weight_averaging, CategoricalHyperparameter) + use_snapshot_ensemble = get_hyperparameter(cs, use_snapshot_ensemble, CategoricalHyperparameter) + cs.add_hyperparameter(use_snapshot_ensemble) + cond = EqualsCondition(se_lastk, use_snapshot_ensemble, True) + cs.add_condition(cond) + + add_hyperparameter(cs, use_lookahead_optimizer, CategoricalHyperparameter) + la_config_space = Lookahead.get_hyperparameter_search_space(la_steps=la_steps, + la_alpha=la_alpha) + parent_hyperparameter = {'parent': use_lookahead_optimizer, 'value': True} + cs.add_configuration_space( + Lookahead.__name__, + la_config_space, + parent_hyperparameter=parent_hyperparameter + ) + if dataset_properties is not None: if STRING_TO_TASK_TYPES[dataset_properties['task_type']] in CLASSIFICATION_TASKS: add_hyperparameter(cs, weighted_loss, CategoricalHyperparameter) diff --git a/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py b/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py index 297959356..8b720731d 100644 --- a/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py +++ b/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py @@ -1,21 +1,30 @@ -import typing +from typing import Any, Callable, Dict, Optional, Tuple +from ConfigSpace.conditions import EqualsCondition from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( CategoricalHyperparameter, + Constant, UniformFloatHyperparameter, ) import numpy as np from autoPyTorch.constants import CLASSIFICATION_TASKS, STRING_TO_TASK_TYPES -from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter +from autoPyTorch.pipeline.components.training.trainer.utils import Lookahead +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter class MixUp: def __init__(self, alpha: float, weighted_loss: bool = False, - random_state: typing.Optional[np.random.RandomState] = None): + random_state: Optional[np.random.RandomState] = None, + use_stochastic_weight_averaging: bool = False, + use_snapshot_ensemble: bool = False, + se_lastk: int = 3, + use_lookahead_optimizer: bool = True, + **lookahead_config: Any + ): """ This class handles the training of a network for a single given epoch. @@ -23,27 +32,79 @@ def __init__(self, alpha: float, alpha (float): the mixup ratio """ + self.use_stochastic_weight_averaging = use_stochastic_weight_averaging self.weighted_loss = weighted_loss - self.alpha = alpha self.random_state = random_state + self.use_snapshot_ensemble = use_snapshot_ensemble + self.se_lastk = se_lastk + self.use_lookahead_optimizer = use_lookahead_optimizer + # Add default values for the lookahead optimizer + if len(lookahead_config) == 0: + lookahead_config = {f'{Lookahead.__name__}:la_steps': 6, + f'{Lookahead.__name__}:la_alpha': 0.6} + self.lookahead_config = lookahead_config + self.alpha = alpha def criterion_preparation(self, y_a: np.ndarray, y_b: np.ndarray = None, lam: float = 1.0 - ) -> typing.Callable: + ) -> Callable: return lambda criterion, pred: lam * criterion(pred, y_a) + (1 - lam) * criterion(pred, y_b) @staticmethod def get_hyperparameter_search_space( - dataset_properties: typing.Optional[typing.Dict] = None, - alpha: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="alpha", - value_range=(0, 1), - default_value=0.2), + dataset_properties: Optional[Dict] = None, weighted_loss: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="weighted_loss", value_range=(True, False), default_value=True), + alpha: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="alpha", + value_range=(0, 1), + default_value=0.2), + la_steps: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="la_steps", + value_range=(5, 10), + default_value=6, + log=False), + la_alpha: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="la_alpha", + value_range=(0.5, 0.8), + default_value=0.6, + log=False), + use_lookahead_optimizer: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="use_lookahead_optimizer", + value_range=(True, False), + default_value=True), + use_stochastic_weight_averaging: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="use_stochastic_weight_averaging", + value_range=(True, False), + default_value=True), + use_snapshot_ensemble: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="use_snapshot_ensemble", + value_range=(True, False), + default_value=True), + se_lastk: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="se_lastk", + value_range=(3,), + default_value=3), ) -> ConfigurationSpace: cs = ConfigurationSpace() add_hyperparameter(cs, alpha, UniformFloatHyperparameter) + add_hyperparameter(cs, se_lastk, Constant) + add_hyperparameter(cs, use_stochastic_weight_averaging, CategoricalHyperparameter) + use_snapshot_ensemble = get_hyperparameter(cs, use_snapshot_ensemble, CategoricalHyperparameter) + cs.add_hyperparameter(use_snapshot_ensemble) + cond = EqualsCondition(se_lastk, use_snapshot_ensemble, True) + cs.add_condition(cond) + + add_hyperparameter(cs, use_lookahead_optimizer, CategoricalHyperparameter) + la_config_space = Lookahead.get_hyperparameter_search_space(la_steps=la_steps, + la_alpha=la_alpha) + parent_hyperparameter = {'parent': use_lookahead_optimizer, 'value': True} + cs.add_configuration_space( + Lookahead.__name__, + la_config_space, + parent_hyperparameter=parent_hyperparameter + ) + if dataset_properties is not None: if STRING_TO_TASK_TYPES[dataset_properties['task_type']] in CLASSIFICATION_TASKS: add_hyperparameter(cs, weighted_loss, CategoricalHyperparameter) diff --git a/autoPyTorch/pipeline/components/training/trainer/utils.py b/autoPyTorch/pipeline/components/training/trainer/utils.py index 10c2aeb9c..ffada50ac 100644 --- a/autoPyTorch/pipeline/components/training/trainer/utils.py +++ b/autoPyTorch/pipeline/components/training/trainer/utils.py @@ -11,6 +11,8 @@ import torch from torch.optim.optimizer import Optimizer +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter + def update_model_state_dict_from_swa(model: torch.nn.Module, swa_state_dict: Dict) -> None: """ @@ -41,7 +43,6 @@ def __init__(self, optimizer: Optimizer, config: Dict[str, Any]) -> None: """optimizer: inner optimizer la_steps (int): number of lookahead steps la_alpha (float): linear interpolation factor. 1.0 recovers the inner optimizer. - pullback_momentum (str): change to inner optimizer momentum on interpolation update """ self.optimizer = optimizer self._la_step = 0 # counter for inner optimizer @@ -148,18 +149,20 @@ def to(self, device: str) -> None: @staticmethod def get_hyperparameter_search_space( - la_steps: Tuple[Tuple, int, bool] = ((5, 10), 6, False), - la_alpha: Tuple[Tuple, float, bool] = ((0.5, 0.8), 0.6, False), + la_steps: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="la_steps", + value_range=(5, 10), + default_value=6, + log=False), + la_alpha: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="la_alpha", + value_range=(0.5, 0.8), + default_value=0.6, + log=False), ) -> ConfigurationSpace: cs = ConfigurationSpace() - la_steps = UniformIntegerHyperparameter('la_steps', lower=la_steps[0][0], - upper=la_steps[0][1], - default_value=la_steps[1], - log=la_steps[2]) - la_alpha = UniformFloatHyperparameter('la_alpha', lower=la_alpha[0][0], - upper=la_alpha[0][1], - default_value=la_alpha[1], - log=la_alpha[2]) - cs.add_hyperparameters([la_steps, la_alpha]) + + add_hyperparameter(cs, la_steps, UniformIntegerHyperparameter) + add_hyperparameter(cs, la_alpha, UniformFloatHyperparameter) return cs diff --git a/test/test_pipeline/components/training/test_training.py b/test/test_pipeline/components/training/test_training.py index 3ddbdacc5..a41c182f7 100644 --- a/test/test_pipeline/components/training/test_training.py +++ b/test/test_pipeline/components/training/test_training.py @@ -363,7 +363,7 @@ def test_every_trainer_is_valid(): trainer_choice = TrainerChoice(dataset_properties={}) # Make sure all components are returned - assert len(trainer_choice.get_components().keys()) == 6 + assert len(trainer_choice.get_components().keys()) == 7 # For every optimizer in the components, make sure # that it complies with the scikit learn estimator. @@ -394,7 +394,7 @@ def test_every_trainer_is_valid(): @pytest.mark.parametrize("test_input,expected", [ - ("tabular_classification", set(['RowCutMixTrainer', 'RowCutOutTrainer'])), + ("tabular_classification", set(['RowCutMixTrainer', 'RowCutOutTrainer', 'AdversarialTrainer'])), ("image_classification", set(['GridCutMixTrainer', 'GridCutOutTrainer'])), ("time_series_classification", set([])), ]) @@ -427,8 +427,12 @@ def test_get_set_config_space(test_input, expected): # Remove the selected_choice string from the parameter # so we can query in the object for it key = key.replace(selected_choice + ':', '') - assert key in vars(trainer_choice.choice) - assert value == trainer_choice.choice.__dict__[key] + if 'Lookahead' in key: + assert key in trainer_choice.choice.__dict__['lookahead_config'].keys() + assert value == trainer_choice.choice.__dict__['lookahead_config'][key] + else: + assert key in vars(trainer_choice.choice) + assert value == trainer_choice.choice.__dict__[key] @pytest.mark.parametrize("cutmix_prob", [1.0, 0.0]) @@ -570,50 +574,5 @@ def test_epoch_training(self): self.fail("Could not overfit a dummy binary classification under 1000 epochs") -<<<<<<< HEAD -======= -class TrainerTest(unittest.TestCase): - def test_every_trainer_is_valid(self): - """ - Makes sure that every trainer is a valid estimator. - That is, we can fully create an object via get/set params. - - This also test that we can properly initialize each one - of them - """ - trainer_choice = TrainerChoice(dataset_properties={}) - - # Make sure all components are returned - self.assertEqual(len(trainer_choice.get_components().keys()), 7) - - # For every optimizer in the components, make sure - # that it complies with the scikit learn estimator. - # This is important because usually components are forked to workers, - # so the set/get params methods should recreate the same object - for name, trainer in trainer_choice.get_components().items(): - config = trainer.get_hyperparameter_search_space().sample_configuration() - estimator = trainer(**config) - estimator_clone = clone(estimator) - estimator_clone_params = estimator_clone.get_params() - - # Make sure all keys are copied properly - for k, v in estimator.get_params().items(): - self.assertIn(k, estimator_clone_params) - - # Make sure the params getter of estimator are honored - klass = estimator.__class__ - new_object_params = estimator.get_params(deep=False) - for name, param in new_object_params.items(): - new_object_params[name] = clone(param, safe=False) - new_object = klass(**new_object_params) - params_set = new_object.get_params(deep=False) - - for name in new_object_params: - param1 = new_object_params[name] - param2 = params_set[name] - self.assertEqual(param1, param2) - - ->>>>>>> Removing duplicate unit test if __name__ == '__main__': unittest.main() diff --git a/test/test_pipeline/test_tabular_classification.py b/test/test_pipeline/test_tabular_classification.py index dc2ec6fcd..24c0be4ee 100644 --- a/test/test_pipeline/test_tabular_classification.py +++ b/test/test_pipeline/test_tabular_classification.py @@ -3,6 +3,7 @@ import unittest import unittest.mock +from ConfigSpace.configuration_space import Configuration from ConfigSpace.hyperparameters import ( CategoricalHyperparameter, UniformFloatHyperparameter, @@ -344,7 +345,6 @@ def test_set_range_search_space_updates(self, fit_dictionary_tabular): # head, units_layer does not exist in the configspace assert 'fully_connected:units_layer' in e.args[0] - def test_set_choices_updates(self, fit_dictionary_tabular): dataset_properties = {'numerical_columns': [1], 'categorical_columns': [2], 'task_type': 'tabular_classification', 'issparse': False, @@ -371,19 +371,32 @@ def test_set_choices_updates(self, fit_dictionary_tabular): search_space_updates=updates) self._assert_pipeline_search_space(pipeline, updates) - - def test_trainer_cocktails(self, fit_dictionary, mocker): # noqa F811 + @pytest.mark.parametrize('lr_scheduler', ['CosineAnnealingWarmRestarts', + 'ReduceLROnPlateau']) + def test_trainer_cocktails(self, fit_dictionary, mocker, lr_scheduler): # noqa F811 fit_dictionary['epochs'] = 10 pipeline = TabularClassificationPipeline( dataset_properties=fit_dictionary['dataset_properties'], - include={'lr_scheduler': ['CosineAnnealingWarmRestarts']}) + include={'lr_scheduler': [lr_scheduler]}) cs = pipeline.get_hyperparameter_search_space() config = cs.get_default_configuration() + trainer = config.get('trainer:__choice__') + config_dict = config.get_dictionary() + config_dict[f'trainer:{trainer}:use_stochastic_weight_averaging'] = True + config_dict[f'trainer:{trainer}:use_snapshot_ensemble'] = True + if not config_dict[f'trainer:{trainer}:use_lookahead_optimizer']: + config_dict[f'trainer:{trainer}:use_lookahead_optimizer'] = True + default_values = Lookahead.get_hyperparameter_search_space().get_default_configuration().get_dictionary() + for key, value in default_values.items(): + config_dict[f'trainer:{trainer}:Lookahead:{key}'] = value + config = Configuration(cs, values=config_dict) + assert lr_scheduler == config.get('lr_scheduler:__choice__') pipeline.set_hyperparameters(config) pipeline.fit(fit_dictionary.copy()) X = pipeline.transform(fit_dictionary.copy()) - assert 'is_cyclic_scheduler' in X and X['is_cyclic_scheduler'] + assert 'is_cyclic_scheduler' in X and \ + (X['is_cyclic_scheduler'] or config.get('lr_scheduler:__choice__') == 'ReduceLROnPlateau') trainer = config.get('trainer:__choice__') assert 'network_snapshots' in X and \ @@ -395,7 +408,13 @@ def test_trainer_cocktails(self, fit_dictionary, mocker): # noqa F811 assert isinstance(pipeline.predict(fit_dictionary['X_train']), np.ndarray) # As SE is True, _predict should be called 3 times assert pipeline.named_steps['network']._predict.call_count == 3 - assert isinstance(pipeline.named_steps['trainer'].choice.optimizer, Lookahead) + + optimizer = pipeline.named_steps['trainer'].choice.optimizer + assert isinstance(optimizer, Lookahead) + + # check if final value of la_step is epochs * num_batches % la_steps + assert optimizer.get_la_step() == fit_dictionary['epochs'] * len(list(X['train_data_loader'].batch_sampler)) \ + % optimizer._total_la_steps @pytest.mark.parametrize("fit_dictionary_tabular", ['iris'], indirect=True) diff --git a/test/utils.py b/test/utils.py new file mode 100644 index 000000000..171d4d052 --- /dev/null +++ b/test/utils.py @@ -0,0 +1,71 @@ +from pathlib import Path + + +class DisplayablePath(object): + display_filename_prefix_middle = '├──' + display_filename_prefix_last = '└──' + display_parent_prefix_middle = ' ' + display_parent_prefix_last = '│ ' + + def __init__(self, path, parent_path, is_last): + self.path = Path(str(path)) + self.parent = parent_path + self.is_last = is_last + if self.parent: + self.depth = self.parent.depth + 1 + else: + self.depth = 0 + + @property + def displayname(self): + if self.path.is_dir(): + return self.path.name + '/' + return self.path.name + + @classmethod + def make_tree(cls, root, parent=None, is_last=False, criteria=None): + root = Path(str(root)) + criteria = criteria or cls._default_criteria + + displayable_root = cls(root, parent, is_last) + yield displayable_root + + children = sorted(list(path + for path in root.iterdir() + if criteria(path)), + key=lambda s: str(s).lower()) + count = 1 + for path in children: + is_last = count == len(children) + if path.is_dir(): + yield from cls.make_tree(path, + parent=displayable_root, + is_last=is_last, + criteria=criteria) + else: + yield cls(path, displayable_root, is_last) + count += 1 + + @classmethod + def _default_criteria(cls, path): + return True + + def displayable(self): + if self.parent is None: + return self.displayname + + _filename_prefix = (self.display_filename_prefix_last + if self.is_last + else self.display_filename_prefix_middle) + + parts = ['{!s} {!s}'.format(_filename_prefix, + self.displayname)] + + parent = self.parent + while parent and parent.parent is not None: + parts.append(self.display_parent_prefix_middle + if parent.is_last + else self.display_parent_prefix_last) + parent = parent.parent + + return ''.join(reversed(parts)) From 6e0d2bb5fea2f81d6a5f37fa7c45309bba82c718 Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Tue, 23 Feb 2021 12:03:34 +0100 Subject: [PATCH 06/50] Fix errors in Adversarial training --- .../training/trainer/AdversarialTrainer.py | 63 ++++++++++++++++--- .../training/trainer/MixUpTrainer.py | 2 +- .../components/training/trainer/__init__.py | 6 ++ .../test_tabular_classification.py | 19 +++--- 4 files changed, 75 insertions(+), 15 deletions(-) diff --git a/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py b/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py index 964d4b993..584fc67ea 100644 --- a/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py @@ -1,9 +1,11 @@ import typing from copy import deepcopy +from ConfigSpace.conditions import EqualsCondition from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( CategoricalHyperparameter, + Constant, UniformFloatHyperparameter, ) @@ -14,7 +16,7 @@ from autoPyTorch.constants import CLASSIFICATION_TASKS, STRING_TO_TASK_TYPES from autoPyTorch.pipeline.components.training.trainer.base_trainer import BaseTrainerComponent -from autoPyTorch.utils.logging_ import PicklableClientLogger +from autoPyTorch.pipeline.components.training.trainer.utils import Lookahead class AdversarialTrainer(BaseTrainerComponent): @@ -23,6 +25,11 @@ def __init__( epsilon: float, weighted_loss: bool = False, random_state: typing.Optional[np.random.RandomState] = None, + use_stochastic_weight_averaging: bool = False, + use_snapshot_ensemble: bool = False, + se_lastk: int = 3, + use_lookahead_optimizer: bool = True, + **lookahead_config: typing.Dict[str, typing.Any] ): """ This class handles the training of a network for a single given epoch. @@ -31,9 +38,14 @@ def __init__( epsilon (float): The perturbation magnitude. """ - super().__init__(random_state=random_state) + super().__init__(random_state=random_state, + weighted_loss=weighted_loss, + use_stochastic_weight_averaging=use_stochastic_weight_averaging, + use_snapshot_ensemble=use_snapshot_ensemble, + se_lastk=se_lastk, + use_lookahead_optimizer=use_lookahead_optimizer, + **lookahead_config) self.epsilon = epsilon - self.weighted_loss = weighted_loss def data_preparation(self, X: np.ndarray, y: np.ndarray, ) -> typing.Tuple[typing.Tuple[np.ndarray, np.ndarray], typing.Dict[str, np.ndarray]]: @@ -142,15 +154,52 @@ def get_properties(dataset_properties: typing.Optional[typing.Dict[str, typing.A } @staticmethod - def get_hyperparameter_search_space(dataset_properties: typing.Optional[typing.Dict] = None, - epsilon: typing.Tuple[typing.Tuple[float, float], float] = ((0.05, 0.2), 0.2), - weighted_loss: typing.Tuple[typing.Tuple, bool] = ((True, False), True) - ) -> ConfigurationSpace: + def get_hyperparameter_search_space( + dataset_properties: typing.Optional[typing.Dict] = None, + weighted_loss: typing.Tuple[typing.Tuple, bool] = ((True, False), True), + use_stochastic_weight_averaging: typing.Tuple[typing.Tuple, bool] = ((True, False), True), + use_snapshot_ensemble: typing.Tuple[typing.Tuple, bool] = ((True, False), True), + se_lastk: typing.Tuple[typing.Tuple, int] = ((3,), 3), + use_lookahead_optimizer: typing.Tuple[typing.Tuple, bool] = ((True, False), True), + la_steps: typing.Tuple[typing.Tuple, int, bool] = ((5, 10), 6, False), + la_alpha: typing.Tuple[typing.Tuple, float, bool] = ((0.5, 0.8), 0.6, False), + epsilon: typing.Tuple[typing.Tuple[float, float], float] = ((0.05, 0.2), 0.2), + ) -> ConfigurationSpace: epsilon = UniformFloatHyperparameter( "epsilon", epsilon[0][0], epsilon[0][1], default_value=epsilon[1]) weighted_loss = CategoricalHyperparameter("weighted_loss", choices=weighted_loss[0], default_value=weighted_loss[1]) + + use_swa = CategoricalHyperparameter("use_stochastic_weight_averaging", + choices=use_stochastic_weight_averaging[0], + default_value=use_stochastic_weight_averaging[1]) + use_se = CategoricalHyperparameter("use_snapshot_ensemble", + choices=use_snapshot_ensemble[0], + default_value=use_snapshot_ensemble[1]) + + # Note, this is not easy to be considered as a hyperparameter. + # When used with cyclic learning rates, it depends on the number + # of restarts. + se_lastk = Constant('se_lastk', se_lastk[1]) + + use_lookahead_optimizer = CategoricalHyperparameter("use_lookahead_optimizer", + choices=use_lookahead_optimizer[0], + default_value=use_lookahead_optimizer[1]) + + config_space = Lookahead.get_hyperparameter_search_space(la_steps=la_steps, + la_alpha=la_alpha) + parent_hyperparameter = {'parent': use_lookahead_optimizer, 'value': True} + cs = ConfigurationSpace() + cs.add_hyperparameters([use_swa, use_se, se_lastk, use_lookahead_optimizer]) + cs.add_configuration_space( + Lookahead.__name__, + config_space, + parent_hyperparameter=parent_hyperparameter + ) + cond = EqualsCondition(se_lastk, use_se, True) + cs.add_condition(cond) + cs.add_hyperparameters([epsilon]) if dataset_properties is not None: if STRING_TO_TASK_TYPES[dataset_properties['task_type']] not in CLASSIFICATION_TASKS: diff --git a/autoPyTorch/pipeline/components/training/trainer/MixUpTrainer.py b/autoPyTorch/pipeline/components/training/trainer/MixUpTrainer.py index be909abfb..b0f9cf696 100644 --- a/autoPyTorch/pipeline/components/training/trainer/MixUpTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/MixUpTrainer.py @@ -1,4 +1,4 @@ -from typing import Any, Callable, Dict, Optional, Tuple, Union +from typing import Any, Dict, Optional, Tuple, Union import numpy as np import torch diff --git a/autoPyTorch/pipeline/components/training/trainer/__init__.py b/autoPyTorch/pipeline/components/training/trainer/__init__.py index dc0c8477a..d4be92f2b 100755 --- a/autoPyTorch/pipeline/components/training/trainer/__init__.py +++ b/autoPyTorch/pipeline/components/training/trainer/__init__.py @@ -198,6 +198,12 @@ def get_hyperparameter_search_space( if default is None: defaults = ['StandardTrainer', + 'AdversarialTrainer', + 'GridCutMixTrainer', + 'GridCutOutTrainer', + 'MixUpTrainer', + 'RowCutMixTrainer', + 'RowCutOutTrainer', ] for default_ in defaults: if default_ in available_trainers: diff --git a/test/test_pipeline/test_tabular_classification.py b/test/test_pipeline/test_tabular_classification.py index 24c0be4ee..240f19799 100644 --- a/test/test_pipeline/test_tabular_classification.py +++ b/test/test_pipeline/test_tabular_classification.py @@ -235,8 +235,8 @@ def test_network_optimizer_lr_handshake(self, fit_dictionary_tabular): # Then fitting a optimizer should fail if no network: assert 'optimizer' in pipeline.named_steps.keys() with pytest.raises( - ValueError, - match=r"To fit .+?, expected fit dictionary to have 'network' but got .*" + ValueError, + match=r"To fit .+?, expected fit dictionary to have 'network' but got .*" ): pipeline.named_steps['optimizer'].fit({'dataset_properties': {}}, None) @@ -248,8 +248,8 @@ def test_network_optimizer_lr_handshake(self, fit_dictionary_tabular): # Then fitting a optimizer should fail if no network: assert 'lr_scheduler' in pipeline.named_steps.keys() with pytest.raises( - ValueError, - match=r"To fit .+?, expected fit dictionary to have 'optimizer' but got .*" + ValueError, + match=r"To fit .+?, expected fit dictionary to have 'optimizer' but got .*" ): pipeline.named_steps['lr_scheduler'].fit({'dataset_properties': {}}, None) @@ -371,16 +371,21 @@ def test_set_choices_updates(self, fit_dictionary_tabular): search_space_updates=updates) self._assert_pipeline_search_space(pipeline, updates) + @pytest.mark.parametrize('trainer', ['StandardTrainer', + 'AdversarialTrainer', + 'MixUpTrainer', + 'RowCutMixTrainer', + 'RowCutOutTrainer']) @pytest.mark.parametrize('lr_scheduler', ['CosineAnnealingWarmRestarts', 'ReduceLROnPlateau']) - def test_trainer_cocktails(self, fit_dictionary, mocker, lr_scheduler): # noqa F811 + def test_trainer_cocktails(self, fit_dictionary, mocker, lr_scheduler, trainer): # noqa F811 fit_dictionary['epochs'] = 10 pipeline = TabularClassificationPipeline( dataset_properties=fit_dictionary['dataset_properties'], - include={'lr_scheduler': [lr_scheduler]}) + include={'lr_scheduler': [lr_scheduler], 'trainer': [trainer]}) cs = pipeline.get_hyperparameter_search_space() config = cs.get_default_configuration() - trainer = config.get('trainer:__choice__') + assert trainer == config.get('trainer:__choice__') config_dict = config.get_dictionary() config_dict[f'trainer:{trainer}:use_stochastic_weight_averaging'] = True config_dict[f'trainer:{trainer}:use_snapshot_ensemble'] = True From 9290b073fb9ba43e8529a4a841712687b5cb4493 Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Tue, 23 Feb 2021 14:46:59 +0100 Subject: [PATCH 07/50] Fix pickling error for swa model --- .../training/trainer/AdversarialTrainer.py | 2 +- .../training/trainer/StandardTrainer.py | 2 +- .../training/trainer/base_trainer.py | 22 +++++++++++++++---- .../components/training/trainer/utils.py | 19 ++++++++++++++++ 4 files changed, 39 insertions(+), 6 deletions(-) diff --git a/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py b/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py index 584fc67ea..81aa76fbb 100644 --- a/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py @@ -29,7 +29,7 @@ def __init__( use_snapshot_ensemble: bool = False, se_lastk: int = 3, use_lookahead_optimizer: bool = True, - **lookahead_config: typing.Dict[str, typing.Any] + **lookahead_config: typing.Any ): """ This class handles the training of a network for a single given epoch. diff --git a/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py b/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py index b3c47eaee..825e1c034 100644 --- a/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py @@ -19,7 +19,7 @@ def __init__(self, weighted_loss: bool = False, se_lastk: int = 3, use_lookahead_optimizer: bool = True, random_state: Optional[Union[np.random.RandomState, int]] = None, - **lookahead_config: Dict[str, Any]): + **lookahead_config: Any): """ This class handles the training of a network for a single given epoch. diff --git a/autoPyTorch/pipeline/components/training/trainer/base_trainer.py b/autoPyTorch/pipeline/components/training/trainer/base_trainer.py index a79d3a74b..6dc908d4d 100644 --- a/autoPyTorch/pipeline/components/training/trainer/base_trainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/base_trainer.py @@ -30,7 +30,7 @@ ) from autoPyTorch.pipeline.components.training.trainer.utils import Lookahead from autoPyTorch.pipeline.components.training.metrics.utils import calculate_score -from autoPyTorch.pipeline.components.training.trainer.utils import Lookahead +from autoPyTorch.pipeline.components.training.trainer.utils import Lookahead, swa_average_function from autoPyTorch.utils.common import FitRequirement from autoPyTorch.utils.implementations import get_loss_weight_strategy @@ -230,7 +230,9 @@ def __init__(self, weighted_loss: bool = True, se_lastk: int = 3, use_lookahead_optimizer: bool = True, random_state: Optional[Union[np.random.RandomState, int]] = None, - **lookahead_config: Dict[str, Any]) -> None: + swa_model: Optional[torch.nn.Module] = None, + model_snapshots: Optional[List[torch.nn.Module]] = None, + **lookahead_config: Any) -> None: if random_state is None: # A trainer components need a random state for # sampling -- for example in MixUp training @@ -243,6 +245,8 @@ def __init__(self, weighted_loss: bool = True, self.use_snapshot_ensemble = use_snapshot_ensemble self.se_lastk = se_lastk self.use_lookahead_optimizer = use_lookahead_optimizer + self.swa_model = swa_model + self.model_snapshots = model_snapshots # Add default values for the lookahead optimizer if len(lookahead_config) == 0: lookahead_config = {f'{Lookahead.__name__}:la_steps': 6, @@ -286,7 +290,7 @@ def prepare( # in case we are using swa, maintain an averaged model, if self.use_stochastic_weight_averaging: - self.swa_model = swa_utils.AveragedModel(self.model) + self.swa_model = swa_utils.AveragedModel(self.model, avg_fn=swa_average_function) # in case we are using se or swa, initialise budget_threshold to know when to start swa or se self._budget_threshold = 0 @@ -297,7 +301,7 @@ def prepare( # in case we are using se, initialise list to store model snapshots if self.use_snapshot_ensemble: - self.model_snapshots: List[torch.nn.Module] = list() + self.model_snapshots = list() # setup the optimizers if self.use_lookahead_optimizer: @@ -335,20 +339,30 @@ def on_epoch_end(self, X: Dict[str, Any], epoch: int) -> bool: if X['is_cyclic_scheduler']: if hasattr(self.scheduler, 'T_cur') and self.scheduler.T_cur == 0 and epoch != 1: if self.use_stochastic_weight_averaging: + assert self.swa_model is not None, "SWA model can't be none when" \ + " stochastic weight averaging is enabled" self.swa_model.update_parameters(self.model) if self.use_snapshot_ensemble: + assert self.model_snapshots is not None, "model snapshots container can't be " \ + "none when snapshot ensembling is enabled" model_copy = deepcopy(self.swa_model) if self.use_stochastic_weight_averaging \ else deepcopy(self.model) + assert model_copy is not None model_copy.cpu() self.model_snapshots.append(model_copy) self.model_snapshots = self.model_snapshots[-self.se_lastk:] else: if epoch > self._budget_threshold: if self.use_stochastic_weight_averaging: + assert self.swa_model is not None, "SWA model can't be none when" \ + " stochastic weight averaging is enabled" self.swa_model.update_parameters(self.model) if self.use_snapshot_ensemble: + assert self.model_snapshots is not None, "model snapshots container can't be " \ + "none when snapshot ensembling is enabled" model_copy = deepcopy(self.swa_model) if self.use_stochastic_weight_averaging \ else deepcopy(self.model) + assert model_copy is not None model_copy.cpu() self.model_snapshots.append(model_copy) self.model_snapshots = self.model_snapshots[-self.se_lastk:] diff --git a/autoPyTorch/pipeline/components/training/trainer/utils.py b/autoPyTorch/pipeline/components/training/trainer/utils.py index ffada50ac..b78e438f3 100644 --- a/autoPyTorch/pipeline/components/training/trainer/utils.py +++ b/autoPyTorch/pipeline/components/training/trainer/utils.py @@ -34,6 +34,25 @@ def update_model_state_dict_from_swa(model: torch.nn.Module, swa_state_dict: Dic model_state[name].copy_(param) +def swa_average_function(averaged_model_parameter: torch.nn.parameter.Parameter, + model_parameter: torch.nn.parameter.Parameter, + num_averaged: int) -> torch.nn.parameter.Parameter: + """ + Pickling the averaged function causes an error because of + how pytorch initialises the average function. + Passing this function fixes the issue. + Args: + averaged_model_parameter: + model_parameter: + num_averaged: + + Returns: + + """ + return averaged_model_parameter + \ + (model_parameter - averaged_model_parameter) / (num_averaged + 1) + + class Lookahead(Optimizer): r"""PyTorch implementation of the lookahead wrapper. Lookahead Optimizer: https://arxiv.org/abs/1907.08610 From c35c795e90ec9d962902898d2b736974ed49a689 Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Tue, 23 Feb 2021 17:51:00 +0100 Subject: [PATCH 08/50] Fix issues after rebase from refactor_development Fix flake --- .../training/trainer/AdversarialTrainer.py | 2 +- .../training/trainer/base_trainer.py | 3 +- .../training/trainer/cutout_utils.py | 2 +- .../training/trainer/mixup_utils.py | 2 +- .../components/training/trainer/utils.py | 2 +- .../components/training/test_training.py | 24 +++++------- .../test_tabular_classification.py | 15 +++---- test/test_pipeline/test_tabular_regression.py | 39 ++++++++++++------- 8 files changed, 49 insertions(+), 40 deletions(-) diff --git a/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py b/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py index 81aa76fbb..166c85481 100644 --- a/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py @@ -202,6 +202,6 @@ def get_hyperparameter_search_space( cs.add_hyperparameters([epsilon]) if dataset_properties is not None: - if STRING_TO_TASK_TYPES[dataset_properties['task_type']] not in CLASSIFICATION_TASKS: + if STRING_TO_TASK_TYPES[dataset_properties['task_type']] in CLASSIFICATION_TASKS: cs.add_hyperparameters([weighted_loss]) return cs diff --git a/autoPyTorch/pipeline/components/training/trainer/base_trainer.py b/autoPyTorch/pipeline/components/training/trainer/base_trainer.py index 6dc908d4d..56ebbdb0c 100644 --- a/autoPyTorch/pipeline/components/training/trainer/base_trainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/base_trainer.py @@ -8,6 +8,7 @@ CategoricalHyperparameter, Constant ) + import numpy as np import pandas as pd @@ -623,7 +624,7 @@ def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, cs.add_condition(cond) if dataset_properties is not None: - if STRING_TO_TASK_TYPES[dataset_properties['task_type']] not in CLASSIFICATION_TASKS: + if STRING_TO_TASK_TYPES[dataset_properties['task_type']] in CLASSIFICATION_TASKS: cs.add_hyperparameters([weighted_loss]) return cs diff --git a/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py b/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py index cc3b03018..e5ef2ee1d 100644 --- a/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py +++ b/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py @@ -1,4 +1,4 @@ -from typing import Any, Callable, Dict, Optional, Tuple +from typing import Any, Callable, Dict, Optional from ConfigSpace.conditions import EqualsCondition from ConfigSpace.configuration_space import ConfigurationSpace diff --git a/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py b/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py index 8b720731d..5b28b756f 100644 --- a/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py +++ b/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py @@ -1,4 +1,4 @@ -from typing import Any, Callable, Dict, Optional, Tuple +from typing import Any, Callable, Dict, Optional from ConfigSpace.conditions import EqualsCondition from ConfigSpace.configuration_space import ConfigurationSpace diff --git a/autoPyTorch/pipeline/components/training/trainer/utils.py b/autoPyTorch/pipeline/components/training/trainer/utils.py index b78e438f3..9193be6a6 100644 --- a/autoPyTorch/pipeline/components/training/trainer/utils.py +++ b/autoPyTorch/pipeline/components/training/trainer/utils.py @@ -1,6 +1,6 @@ import re from collections import defaultdict -from typing import Any, Callable, Dict, List, Optional, Tuple +from typing import Any, Callable, Dict, List, Optional from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( diff --git a/test/test_pipeline/components/training/test_training.py b/test/test_pipeline/components/training/test_training.py index a41c182f7..4cc4efe29 100644 --- a/test/test_pipeline/components/training/test_training.py +++ b/test/test_pipeline/components/training/test_training.py @@ -547,26 +547,20 @@ def test_epoch_training(self): Makes sure we are able to train a model and produce good training performance """ - trainer = AdversarialTrainer(epsilon=0.07) - trainer.prepare( - scheduler=None, - model=self.model, - metrics=self.metrics, - criterion=self.criterion, - budget_tracker=self.budget_tracker, - optimizer=self.optimizer, - device=self.device, - metrics_during_training=True, - task_type=self.task_type, - output_type=self.output_type, - labels=self.y - ) + (trainer, + _, + _, + loader, + _, + epochs, + logger) = self.prepare_trainer(AdversarialTrainer(epsilon=0.07), + constants.TABULAR_CLASSIFICATION) # Train the model counter = 0 accuracy = 0 while accuracy < 0.7: - loss, metrics = trainer.train_epoch(self.loader, epoch=1, logger=self.logger, writer=None) + loss, metrics = trainer.train_epoch(loader, epoch=1, writer=None) counter += 1 accuracy = metrics['accuracy'] diff --git a/test/test_pipeline/test_tabular_classification.py b/test/test_pipeline/test_tabular_classification.py index 240f19799..d612535c3 100644 --- a/test/test_pipeline/test_tabular_classification.py +++ b/test/test_pipeline/test_tabular_classification.py @@ -378,10 +378,10 @@ def test_set_choices_updates(self, fit_dictionary_tabular): 'RowCutOutTrainer']) @pytest.mark.parametrize('lr_scheduler', ['CosineAnnealingWarmRestarts', 'ReduceLROnPlateau']) - def test_trainer_cocktails(self, fit_dictionary, mocker, lr_scheduler, trainer): # noqa F811 - fit_dictionary['epochs'] = 10 + def test_trainer_cocktails(self, fit_dictionary_tabular, mocker, lr_scheduler, trainer): # noqa F811 + fit_dictionary_tabular['epochs'] = 10 pipeline = TabularClassificationPipeline( - dataset_properties=fit_dictionary['dataset_properties'], + dataset_properties=fit_dictionary_tabular['dataset_properties'], include={'lr_scheduler': [lr_scheduler], 'trainer': [trainer]}) cs = pipeline.get_hyperparameter_search_space() config = cs.get_default_configuration() @@ -398,8 +398,8 @@ def test_trainer_cocktails(self, fit_dictionary, mocker, lr_scheduler, trainer): assert lr_scheduler == config.get('lr_scheduler:__choice__') pipeline.set_hyperparameters(config) - pipeline.fit(fit_dictionary.copy()) - X = pipeline.transform(fit_dictionary.copy()) + pipeline.fit(fit_dictionary_tabular.copy()) + X = pipeline.transform(fit_dictionary_tabular.copy()) assert 'is_cyclic_scheduler' in X and \ (X['is_cyclic_scheduler'] or config.get('lr_scheduler:__choice__') == 'ReduceLROnPlateau') @@ -410,7 +410,7 @@ def test_trainer_cocktails(self, fit_dictionary, mocker, lr_scheduler, trainer): mocker.patch("autoPyTorch.pipeline.components.setup.network.base_network.NetworkComponent._predict", return_value=torch.Tensor([1])) # Assert that predict gives no error when swa and se are on - assert isinstance(pipeline.predict(fit_dictionary['X_train']), np.ndarray) + assert isinstance(pipeline.predict(fit_dictionary_tabular['X_train']), np.ndarray) # As SE is True, _predict should be called 3 times assert pipeline.named_steps['network']._predict.call_count == 3 @@ -418,7 +418,8 @@ def test_trainer_cocktails(self, fit_dictionary, mocker, lr_scheduler, trainer): assert isinstance(optimizer, Lookahead) # check if final value of la_step is epochs * num_batches % la_steps - assert optimizer.get_la_step() == fit_dictionary['epochs'] * len(list(X['train_data_loader'].batch_sampler)) \ + assert optimizer.get_la_step() == fit_dictionary_tabular['epochs'] * \ + len(list(X['train_data_loader'].batch_sampler)) \ % optimizer._total_la_steps diff --git a/test/test_pipeline/test_tabular_regression.py b/test/test_pipeline/test_tabular_regression.py index c6c475b91..bc35ac796 100644 --- a/test/test_pipeline/test_tabular_regression.py +++ b/test/test_pipeline/test_tabular_regression.py @@ -58,8 +58,10 @@ def _assert_pipeline_search_space(self, pipeline, search_space_updates): def test_pipeline_fit(self, fit_dictionary_tabular): """This test makes sure that the pipeline is able to fit given random combinations of hyperparameters across the pipeline""" + # TODO: fix issue where adversarial also works for regression pipeline = TabularRegressionPipeline( - dataset_properties=fit_dictionary_tabular['dataset_properties']) + dataset_properties=fit_dictionary_tabular['dataset_properties'], + exclude={'trainer': ['AdversarialTrainer']}) cs = pipeline.get_hyperparameter_search_space() config = cs.sample_configuration() @@ -84,7 +86,8 @@ def test_pipeline_predict(self, fit_dictionary_tabular): given a random configuration""" X = fit_dictionary_tabular['X_train'].copy() pipeline = TabularRegressionPipeline( - dataset_properties=fit_dictionary_tabular['dataset_properties']) + dataset_properties=fit_dictionary_tabular['dataset_properties'], + exclude={'trainer': ['AdversarialTrainer']}) cs = pipeline.get_hyperparameter_search_space() config = cs.sample_configuration() @@ -112,7 +115,8 @@ def test_pipeline_transform(self, fit_dictionary_tabular): """ pipeline = TabularRegressionPipeline( - dataset_properties=fit_dictionary_tabular['dataset_properties']) + dataset_properties=fit_dictionary_tabular['dataset_properties'], + exclude={'trainer': ['AdversarialTrainer']}) cs = pipeline.get_hyperparameter_search_space() config = cs.sample_configuration() pipeline.set_hyperparameters(config) @@ -148,7 +152,8 @@ def test_default_configuration(self, fit_dictionary_tabular, is_small_preprocess fit_dictionary_tabular['is_small_preprocess'] = is_small_preprocess pipeline = TabularRegressionPipeline( - dataset_properties=fit_dictionary_tabular['dataset_properties']) + dataset_properties=fit_dictionary_tabular['dataset_properties'], + exclude={'trainer': ['AdversarialTrainer']}) with unittest.mock.patch.object(pipeline.named_steps['trainer'].choice, 'train_epoch') \ as patch_train: @@ -158,8 +163,9 @@ def test_default_configuration(self, fit_dictionary_tabular, is_small_preprocess def test_remove_key_check_requirements(self, fit_dictionary_tabular): """Makes sure that when a key is removed from X, correct error is outputted""" pipeline = TabularRegressionPipeline( - dataset_properties=fit_dictionary_tabular['dataset_properties']) - for key in ['num_run', 'device', 'split_id', 'torch_num_threads', 'dataset_properties']: + dataset_properties=fit_dictionary_tabular['dataset_properties'], + exclude={'trainer': ['AdversarialTrainer']}) + for key in ['num_run', 'device', 'split_id', 'use_pynisher', 'torch_num_threads', 'dataset_properties']: fit_dictionary_tabular_copy = fit_dictionary_tabular.copy() fit_dictionary_tabular_copy.pop(key) with pytest.raises(ValueError, match=r"To fit .+?, expected fit dictionary to have"): @@ -169,7 +175,8 @@ def test_network_optimizer_lr_handshake(self, fit_dictionary_tabular): """Fitting a network should put the network in the X""" # Create the pipeline to check. A random config should be sufficient pipeline = TabularRegressionPipeline( - dataset_properties=fit_dictionary_tabular['dataset_properties']) + dataset_properties=fit_dictionary_tabular['dataset_properties'], + exclude={'trainer': ['AdversarialTrainer']}) cs = pipeline.get_hyperparameter_search_space() config = cs.sample_configuration() pipeline.set_hyperparameters(config) @@ -212,7 +219,8 @@ def test_network_optimizer_lr_handshake(self, fit_dictionary_tabular): def test_get_fit_requirements(self, fit_dictionary_tabular): dataset_properties = {'numerical_columns': [], 'categorical_columns': [], 'task_type': 'tabular_regression'} - pipeline = TabularRegressionPipeline(dataset_properties=dataset_properties) + pipeline = TabularRegressionPipeline(dataset_properties=dataset_properties, + exclude={'trainer': ['AdversarialTrainer']}) fit_requirements = pipeline.get_fit_requirements() # check if fit requirements is a list of FitRequirement named tuples @@ -224,7 +232,8 @@ def test_apply_search_space_updates(self, fit_dictionary_tabular, search_space_u dataset_properties = {'numerical_columns': [1], 'categorical_columns': [2], 'task_type': 'tabular_regression'} pipeline = TabularRegressionPipeline(dataset_properties=dataset_properties, - search_space_updates=search_space_updates) + search_space_updates=search_space_updates, + exclude={'trainer': ['AdversarialTrainer']}) self._assert_pipeline_search_space(pipeline, search_space_updates) def test_read_and_update_search_space(self, fit_dictionary_tabular, search_space_updates): @@ -241,7 +250,8 @@ def test_read_and_update_search_space(self, fit_dictionary_tabular, search_space dataset_properties = {'numerical_columns': [1], 'categorical_columns': [2], 'task_type': 'tabular_regression'} pipeline = TabularRegressionPipeline(dataset_properties=dataset_properties, - search_space_updates=file_search_space_updates) + search_space_updates=file_search_space_updates, + exclude={'trainer': ['AdversarialTrainer']}) assert file_search_space_updates == pipeline.search_space_updates def test_error_search_space_updates(self, fit_dictionary_tabular, error_search_space_updates): @@ -249,7 +259,8 @@ def test_error_search_space_updates(self, fit_dictionary_tabular, error_search_s 'task_type': 'tabular_regression'} try: _ = TabularRegressionPipeline(dataset_properties=dataset_properties, - search_space_updates=error_search_space_updates) + search_space_updates=error_search_space_updates, + exclude={'trainer': ['AdversarialTrainer']}) except Exception as e: assert isinstance(e, ValueError) assert re.match(r'Unknown hyperparameter for component .*?\. Expected update ' @@ -258,7 +269,8 @@ def test_error_search_space_updates(self, fit_dictionary_tabular, error_search_s def test_set_range_search_space_updates(self, fit_dictionary_tabular): dataset_properties = {'numerical_columns': [1], 'categorical_columns': [2], 'task_type': 'tabular_regression'} - config_dict = TabularRegressionPipeline(dataset_properties=dataset_properties). \ + config_dict = TabularRegressionPipeline(dataset_properties=dataset_properties, + exclude={'trainer': ['AdversarialTrainer']}). \ get_hyperparameter_search_space()._hyperparameters updates = HyperparameterSearchSpaceUpdates() for i, (name, hyperparameter) in enumerate(config_dict.items()): @@ -278,7 +290,8 @@ def test_set_range_search_space_updates(self, fit_dictionary_tabular): updates.append(node_name=name[0], hyperparameter=hyperparameter_name, value_range=value_range, default_value=default_value) pipeline = TabularRegressionPipeline(dataset_properties=dataset_properties, - search_space_updates=updates) + search_space_updates=updates, + exclude={'trainer': ['AdversarialTrainer']}) try: self._assert_pipeline_search_space(pipeline, updates) From 2512709ce198d6500083f9bb1a7ab8f2c5a6f54f Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Mon, 1 Mar 2021 12:44:54 +0100 Subject: [PATCH 09/50] Added n_restarts as hyperparameter for CosineAnnealing --- .../CosineAnnealingWarmRestarts.py | 39 ++++++++----------- 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingWarmRestarts.py b/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingWarmRestarts.py index ccb58b61d..989b8c4c5 100644 --- a/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingWarmRestarts.py +++ b/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingWarmRestarts.py @@ -1,10 +1,8 @@ -from typing import Any, Dict, Optional, Union +import math +from typing import Any, Dict, Optional, Tuple, Union from ConfigSpace.configuration_space import ConfigurationSpace -from ConfigSpace.hyperparameters import ( - UniformFloatHyperparameter, - UniformIntegerHyperparameter -) +from ConfigSpace.hyperparameters import UniformIntegerHyperparameter import numpy as np @@ -31,14 +29,12 @@ class CosineAnnealingWarmRestarts(BaseLRComponent): def __init__( self, - T_0: int, - T_mult: int, + n_restarts: int, step_interval: Union[str, StepIntervalUnit] = StepIntervalUnit.epoch, - random_state: Optional[np.random.RandomState] = None, + random_state: Optional[np.random.RandomState] = None ): super().__init__(step_interval) - self.T_0 = T_0 - self.T_mult = T_mult + self.n_restarts = n_restarts self.random_state = random_state def fit(self, X: Dict[str, Any], y: Any = None) -> BaseLRComponent: @@ -56,10 +52,14 @@ def fit(self, X: Dict[str, Any], y: Any = None) -> BaseLRComponent: # Make sure there is an optimizer self.check_requirements(X, y) + # initialise required attributes for the scheduler + T_mult: int = 1 + T_0: int = math.floor(X['epochs'] / self.n_restarts) + self.scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts( optimizer=X['optimizer'], - T_0=int(self.T_0), - T_mult=int(self.T_mult), + T_0=T_0, + T_mult=T_mult, ) return self @@ -75,18 +75,13 @@ def get_properties(dataset_properties: Optional[Dict[str, BaseDatasetPropertiesT @staticmethod def get_hyperparameter_search_space( dataset_properties: Optional[Dict[str, BaseDatasetPropertiesType]] = None, - T_0: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='T_0', - value_range=(1, 20), - default_value=1, - ), - T_mult: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='T_mult', - value_range=(1.0, 2.0), - default_value=1.0, - ) + n_restarts: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='n_restarts', + value_range=(1, 6), + default_value=3, + ), ) -> ConfigurationSpace: cs = ConfigurationSpace() - add_hyperparameter(cs, T_0, UniformIntegerHyperparameter) - add_hyperparameter(cs, T_mult, UniformFloatHyperparameter) + add_hyperparameter(cs, n_restarts, UniformIntegerHyperparameter) return cs From 41677fdfd217c5d37c6b332580c9f2da124ab35b Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Mon, 1 Mar 2021 15:51:29 +0100 Subject: [PATCH 10/50] fix bug with ealy stopping and swa --- autoPyTorch/pipeline/components/training/trainer/__init__.py | 2 +- .../pipeline/components/training/trainer/base_trainer.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/autoPyTorch/pipeline/components/training/trainer/__init__.py b/autoPyTorch/pipeline/components/training/trainer/__init__.py index d4be92f2b..84f6dacf5 100755 --- a/autoPyTorch/pipeline/components/training/trainer/__init__.py +++ b/autoPyTorch/pipeline/components/training/trainer/__init__.py @@ -450,7 +450,7 @@ def _fit(self, X: Dict[str, Any], y: Any = None, **kwargs: Any) -> 'TrainerChoic if self.run_summary.is_empty(): raise RuntimeError("Budget exhausted without finishing an epoch.") - if self.choice.use_stochastic_weight_averaging: + if self.choice.use_stochastic_weight_averaging and self.choice.swa_updated: # update batch norm statistics swa_utils.update_bn(X['train_data_loader'], self.choice.swa_model.double()) # change model diff --git a/autoPyTorch/pipeline/components/training/trainer/base_trainer.py b/autoPyTorch/pipeline/components/training/trainer/base_trainer.py index 56ebbdb0c..8c949e3ef 100644 --- a/autoPyTorch/pipeline/components/training/trainer/base_trainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/base_trainer.py @@ -304,6 +304,11 @@ def prepare( if self.use_snapshot_ensemble: self.model_snapshots = list() + # in case we are using, swa or se with early stopping, + # we need to make sure network params are only updated + # from the swa model if the swa model was actually updated + self.swa_updated: bool = False + # setup the optimizers if self.use_lookahead_optimizer: optimizer = Lookahead(optimizer=optimizer, config=self.lookahead_config) From e236df0134be9911c069bc0474bfba50095f1b14 Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Mon, 1 Mar 2021 15:52:45 +0100 Subject: [PATCH 11/50] cont... add min for 1 Addressed comments from shuhei, better documentation --- .../setup/lr_scheduler/CosineAnnealingWarmRestarts.py | 8 ++++---- .../pipeline/components/training/trainer/base_trainer.py | 7 ++++++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingWarmRestarts.py b/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingWarmRestarts.py index 989b8c4c5..741402022 100644 --- a/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingWarmRestarts.py +++ b/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingWarmRestarts.py @@ -1,4 +1,3 @@ -import math from typing import Any, Dict, Optional, Tuple, Union from ConfigSpace.configuration_space import ConfigurationSpace @@ -22,8 +21,9 @@ class CosineAnnealingWarmRestarts(BaseLRComponent): restarts in SGDR Args: - T_0 (int): Number of iterations for the first restart - T_mult (int): A factor increases T_{i} after a restart + n_restarts (int): Number of restarts. In autopytorch, based + on the total budget(epochs) there are 'n_restarts' + restarts made periodically. random_state (Optional[np.random.RandomState]): random state """ @@ -54,7 +54,7 @@ def fit(self, X: Dict[str, Any], y: Any = None) -> BaseLRComponent: # initialise required attributes for the scheduler T_mult: int = 1 - T_0: int = math.floor(X['epochs'] / self.n_restarts) + T_0: int = max(X['epochs'] // self.n_restarts, 1) self.scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts( optimizer=X['optimizer'], diff --git a/autoPyTorch/pipeline/components/training/trainer/base_trainer.py b/autoPyTorch/pipeline/components/training/trainer/base_trainer.py index 8c949e3ef..207fbf0dd 100644 --- a/autoPyTorch/pipeline/components/training/trainer/base_trainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/base_trainer.py @@ -216,7 +216,10 @@ class BaseTrainerComponent(autoPyTorchTrainingComponent): weighted_loss (bool, default=True): In case for classification, whether to weight the loss function according to the distribution of classes in the target use_stochastic_weight_averaging (bool, default=True): whether to use stochastic - weight averaging + weight averaging. Stochastic weight averaging is a simple average of + multiple points(model parameters) along the trajectory of SGD. SWA + has been proposed in + [Averaging Weights Leads to Wider Optima and Better Generalization](https://arxiv.org/abs/1803.05407) use_snapshot_ensemble (bool, default=True): whether to use snapshot ensemble se_lastk (int, default=3): Number of snapshots of the network to maintain @@ -348,6 +351,7 @@ def on_epoch_end(self, X: Dict[str, Any], epoch: int) -> bool: assert self.swa_model is not None, "SWA model can't be none when" \ " stochastic weight averaging is enabled" self.swa_model.update_parameters(self.model) + self.swa_updated = True if self.use_snapshot_ensemble: assert self.model_snapshots is not None, "model snapshots container can't be " \ "none when snapshot ensembling is enabled" @@ -363,6 +367,7 @@ def on_epoch_end(self, X: Dict[str, Any], epoch: int) -> bool: assert self.swa_model is not None, "SWA model can't be none when" \ " stochastic weight averaging is enabled" self.swa_model.update_parameters(self.model) + self.swa_updated = True if self.use_snapshot_ensemble: assert self.model_snapshots is not None, "model snapshots container can't be " \ "none when snapshot ensembling is enabled" From f29a7e0ee2c9b224ddd56acb9819e174c0108a05 Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Sun, 7 Mar 2021 15:56:41 +0100 Subject: [PATCH 12/50] Addressed comments from arlind, change in T_mul and T_0 calculations Added debug information for API Fix flake Fix import made test deterministic for feature preprocessing Fix bug in parsing log convert to int Fix bug in testing --- autoPyTorch/evaluation/abstract_evaluator.py | 3 +- .../CosineAnnealingWarmRestarts.py | 9 +-- .../setup/lr_scheduler/ReduceLROnPlateau.py | 1 - test/test_api/api_utils.py | 42 +++++++++++ test/test_api/test_api.py | 5 +- .../test_feature_preprocessor.py | 2 +- .../test_tabular_classification.py | 3 +- test/utils.py | 71 ------------------- 8 files changed, 55 insertions(+), 81 deletions(-) create mode 100644 test/test_api/api_utils.py delete mode 100644 test/utils.py diff --git a/autoPyTorch/evaluation/abstract_evaluator.py b/autoPyTorch/evaluation/abstract_evaluator.py index d20a96b75..d1bd3c43e 100644 --- a/autoPyTorch/evaluation/abstract_evaluator.py +++ b/autoPyTorch/evaluation/abstract_evaluator.py @@ -948,8 +948,7 @@ def file_output( pipeline = None else: pipeline = None - - self.logger.debug("Saving directory {}, {}, {}".format(self.seed, self.num_run, self.budget)) + self.logger.debug("Saving model {}_{}_{} to disk".format(self.seed, self.num_run, self.budget)) self.backend.save_numrun_to_dir( seed=int(self.seed), idx=int(self.num_run), diff --git a/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingWarmRestarts.py b/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingWarmRestarts.py index 741402022..e46248b92 100644 --- a/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingWarmRestarts.py +++ b/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingWarmRestarts.py @@ -53,13 +53,14 @@ def fit(self, X: Dict[str, Any], y: Any = None) -> BaseLRComponent: self.check_requirements(X, y) # initialise required attributes for the scheduler - T_mult: int = 1 - T_0: int = max(X['epochs'] // self.n_restarts, 1) + T_mult: int = 2 + # using Epochs = T_0 * (T_mul ** n_restarts -1) / (T_mul - 1) (Sum of GP) + T_0: int = max((X['epochs'] * (T_mult - 1)) // (T_mult ** self.n_restarts - 1), 1) self.scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts( optimizer=X['optimizer'], - T_0=T_0, - T_mult=T_mult, + T_0=int(T_0), + T_mult=int(T_mult), ) return self diff --git a/autoPyTorch/pipeline/components/setup/lr_scheduler/ReduceLROnPlateau.py b/autoPyTorch/pipeline/components/setup/lr_scheduler/ReduceLROnPlateau.py index 00503cb7e..ed0702796 100644 --- a/autoPyTorch/pipeline/components/setup/lr_scheduler/ReduceLROnPlateau.py +++ b/autoPyTorch/pipeline/components/setup/lr_scheduler/ReduceLROnPlateau.py @@ -99,7 +99,6 @@ def get_hyperparameter_search_space( default_value=0.1, ) ) -> ConfigurationSpace: - cs = ConfigurationSpace() add_hyperparameter(cs, mode, CategoricalHyperparameter) diff --git a/test/test_api/api_utils.py b/test/test_api/api_utils.py new file mode 100644 index 000000000..b355aa802 --- /dev/null +++ b/test/test_api/api_utils.py @@ -0,0 +1,42 @@ +import glob +import os + + +def print_debug_information(automl): + + # Log file path + log_file = glob.glob(os.path.join( + automl._backend.temporary_directory, 'AutoPyTorch*.log'))[0] + + include_messages = ['INFO', 'DEBUG', 'WARN', + 'CRITICAL', 'ERROR', 'FATAL'] + + # There is a lot of content in the log files. Only + # parsing the main message and ignore the metalearning + # messages + try: + with open(log_file) as logfile: + content = logfile.readlines() + + # Get the messages to debug easier! + content = [line for line in content if any( + msg in line for msg in include_messages + ) and 'metalearning' not in line] + + except Exception as e: + return str(e) + + # Also add the run history if any + if hasattr(automl, 'runhistory') and hasattr(automl.runhistory, 'data'): + for k, v in automl.runhistory_.data.items(): + content += ["{}->{}".format(k, v)] + else: + content += ['No RunHistory'] + + # Also add the ensemble history if any + if len(automl.ensemble_performance_history) > 0: + content += [str(h) for h in automl.ensemble_performance_history] + else: + content += ['No Ensemble History'] + + return os.linesep.join(content) diff --git a/test/test_api/test_api.py b/test/test_api/test_api.py index 465d74c6b..3c9bbd1a7 100644 --- a/test/test_api/test_api.py +++ b/test/test_api/test_api.py @@ -41,6 +41,8 @@ from autoPyTorch.pipeline.components.setup.traditional_ml.traditional_learner import _traditional_learners from autoPyTorch.pipeline.components.training.metrics.metrics import accuracy +from test.test_api.api_utils import print_debug_information + CV_NUM_SPLITS = 2 HOLDOUT_NUM_SPLITS = 1 @@ -154,7 +156,8 @@ def test_tabular_classification(openml_id, resampling_strategy, backend, resampl run_key_model_run_dir, f"{estimator.seed}.{successful_num_run}.{run_key.budget}.cv_model" ) - assert os.path.exists(model_file), model_file + time.sleep(5) + assert os.path.exists(model_file), print_debug_information(estimator) model = estimator._backend.load_cv_model_by_seed_and_id_and_budget( estimator.seed, successful_num_run, run_key.budget) diff --git a/test/test_pipeline/components/preprocessing/test_feature_preprocessor.py b/test/test_pipeline/components/preprocessing/test_feature_preprocessor.py index c4c03641c..494601427 100644 --- a/test/test_pipeline/components/preprocessing/test_feature_preprocessor.py +++ b/test/test_pipeline/components/preprocessing/test_feature_preprocessor.py @@ -107,7 +107,7 @@ def test_pipeline_fit_include(self, fit_dictionary_tabular, preprocessor): dataset_properties=fit_dictionary_tabular['dataset_properties'], include={'feature_preprocessor': [preprocessor]}) cs = pipeline.get_hyperparameter_search_space() - config = cs.sample_configuration() + config = cs.get_default_configuration() pipeline.set_hyperparameters(config) try: pipeline.fit(fit_dictionary_tabular) diff --git a/test/test_pipeline/test_tabular_classification.py b/test/test_pipeline/test_tabular_classification.py index d612535c3..3569be2ad 100644 --- a/test/test_pipeline/test_tabular_classification.py +++ b/test/test_pipeline/test_tabular_classification.py @@ -379,7 +379,8 @@ def test_set_choices_updates(self, fit_dictionary_tabular): @pytest.mark.parametrize('lr_scheduler', ['CosineAnnealingWarmRestarts', 'ReduceLROnPlateau']) def test_trainer_cocktails(self, fit_dictionary_tabular, mocker, lr_scheduler, trainer): # noqa F811 - fit_dictionary_tabular['epochs'] = 10 + fit_dictionary_tabular['epochs'] = 20 + fit_dictionary_tabular['early_stopping'] = 20 pipeline = TabularClassificationPipeline( dataset_properties=fit_dictionary_tabular['dataset_properties'], include={'lr_scheduler': [lr_scheduler], 'trainer': [trainer]}) diff --git a/test/utils.py b/test/utils.py deleted file mode 100644 index 171d4d052..000000000 --- a/test/utils.py +++ /dev/null @@ -1,71 +0,0 @@ -from pathlib import Path - - -class DisplayablePath(object): - display_filename_prefix_middle = '├──' - display_filename_prefix_last = '└──' - display_parent_prefix_middle = ' ' - display_parent_prefix_last = '│ ' - - def __init__(self, path, parent_path, is_last): - self.path = Path(str(path)) - self.parent = parent_path - self.is_last = is_last - if self.parent: - self.depth = self.parent.depth + 1 - else: - self.depth = 0 - - @property - def displayname(self): - if self.path.is_dir(): - return self.path.name + '/' - return self.path.name - - @classmethod - def make_tree(cls, root, parent=None, is_last=False, criteria=None): - root = Path(str(root)) - criteria = criteria or cls._default_criteria - - displayable_root = cls(root, parent, is_last) - yield displayable_root - - children = sorted(list(path - for path in root.iterdir() - if criteria(path)), - key=lambda s: str(s).lower()) - count = 1 - for path in children: - is_last = count == len(children) - if path.is_dir(): - yield from cls.make_tree(path, - parent=displayable_root, - is_last=is_last, - criteria=criteria) - else: - yield cls(path, displayable_root, is_last) - count += 1 - - @classmethod - def _default_criteria(cls, path): - return True - - def displayable(self): - if self.parent is None: - return self.displayname - - _filename_prefix = (self.display_filename_prefix_last - if self.is_last - else self.display_filename_prefix_middle) - - parts = ['{!s} {!s}'.format(_filename_prefix, - self.displayname)] - - parent = self.parent - while parent and parent.parent is not None: - parts.append(self.display_parent_prefix_middle - if parent.is_last - else self.display_parent_prefix_last) - parent = parent.parent - - return ''.join(reversed(parts)) From f141d0680e9596366dd3d95da7ddb3e82de00c71 Mon Sep 17 00:00:00 2001 From: Arlind Kadra Date: Tue, 6 Apr 2021 15:28:05 +0200 Subject: [PATCH 13/50] Updating search space (#156) * Updating search space * fix typo * Bug fix * Fixing buggy implementation of predict when using gpu bug fixes fixing code style checks bug fix for use_pynisher in the base pipeline bug fix --- autoPyTorch/pipeline/base_pipeline.py | 4 +- .../CosineAnnealingWarmRestarts.py | 2 +- .../setup/lr_scheduler/ReduceLROnPlateau.py | 3 +- .../components/setup/network/base_network.py | 5 +- .../setup/network_backbone/MLPBackbone.py | 3 +- .../setup/network_backbone/ResNetBackbone.py | 4 +- .../network_backbone/ShapedMLPBackbone.py | 5 +- .../network_backbone/ShapedResNetBackbone.py | 9 +- .../setup/optimizer/AdamOptimizer.py | 11 +- .../setup/optimizer/AdamWOptimizer.py | 5 +- .../setup/optimizer/RMSpropOptimizer.py | 6 +- .../setup/optimizer/SGDOptimizer.py | 5 +- .../training/data_loader/base_data_loader.py | 4 +- .../training/trainer/AdversarialTrainer.py | 109 ++++++++++-------- .../training/trainer/base_trainer.py | 86 ++++++++------ .../training/trainer/cutout_utils.py | 25 ++-- .../training/trainer/mixup_utils.py | 16 +-- test/conftest.py | 1 + 18 files changed, 175 insertions(+), 128 deletions(-) diff --git a/autoPyTorch/pipeline/base_pipeline.py b/autoPyTorch/pipeline/base_pipeline.py index 5c580dbd6..52b5f7579 100644 --- a/autoPyTorch/pipeline/base_pipeline.py +++ b/autoPyTorch/pipeline/base_pipeline.py @@ -527,7 +527,7 @@ def get_fit_requirements(self) -> List[FitRequirement]: Returns: List[NamedTuple]: List of FitRequirements """ - fit_requirements = list() # List[FitRequirement] + fit_requirements: List[FitRequirement] = list() for name, step in self.steps: step_requirements = step.get_fit_requirements() if step_requirements: @@ -596,6 +596,7 @@ def get_pipeline_representation(self) -> Dict[str, str]: @staticmethod def get_default_pipeline_options() -> Dict[str, Any]: + return { 'num_run': 0, 'device': 'cpu', @@ -605,5 +606,6 @@ def get_default_pipeline_options() -> Dict[str, Any]: 'torch_num_threads': 1, 'early_stopping': 10, 'use_tensorboard_logger': True, + 'use_pynisher': False, 'metrics_during_training': True } diff --git a/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingWarmRestarts.py b/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingWarmRestarts.py index e46248b92..46e3fdd26 100644 --- a/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingWarmRestarts.py +++ b/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingWarmRestarts.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, Optional, Union from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import UniformIntegerHyperparameter diff --git a/autoPyTorch/pipeline/components/setup/lr_scheduler/ReduceLROnPlateau.py b/autoPyTorch/pipeline/components/setup/lr_scheduler/ReduceLROnPlateau.py index ed0702796..490d6709f 100644 --- a/autoPyTorch/pipeline/components/setup/lr_scheduler/ReduceLROnPlateau.py +++ b/autoPyTorch/pipeline/components/setup/lr_scheduler/ReduceLROnPlateau.py @@ -1,4 +1,5 @@ -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, Optional, Union + from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( CategoricalHyperparameter, diff --git a/autoPyTorch/pipeline/components/setup/network/base_network.py b/autoPyTorch/pipeline/components/setup/network/base_network.py index daba6307d..cb981e131 100644 --- a/autoPyTorch/pipeline/components/setup/network/base_network.py +++ b/autoPyTorch/pipeline/components/setup/network/base_network.py @@ -128,6 +128,7 @@ def predict(self, loader: torch.utils.data.DataLoader) -> torch.Tensor: return Y_snapshot_preds_tensor.mean(dim=0).cpu().numpy() def _predict(self, network: torch.nn.Module, loader: torch.utils.data.DataLoader) -> torch.Tensor: + network.to(self.device) network.float() network.eval() # Batch prediction @@ -136,10 +137,10 @@ def _predict(self, network: torch.nn.Module, loader: torch.utils.data.DataLoader for i, (X_batch, Y_batch) in enumerate(loader): # Predict on batch X_batch = X_batch.float().to(self.device) - Y_batch_pred = network(X_batch).detach().cpu() + Y_batch_pred = network(X_batch) if self.final_activation is not None: Y_batch_pred = self.final_activation(Y_batch_pred) - Y_batch_preds.append(Y_batch_pred) + Y_batch_preds.append(Y_batch_pred.detach().cpu()) return torch.cat(Y_batch_preds, 0) diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/MLPBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/MLPBackbone.py index f2ed459c3..c8777b032 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/MLPBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/MLPBackbone.py @@ -91,13 +91,13 @@ def get_hyperparameter_search_space( num_units: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="num_units", value_range=(10, 1024), default_value=200, + log=True ), dropout: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="dropout", value_range=(0, 0.8), default_value=0.5, ), ) -> ConfigurationSpace: - cs = ConfigurationSpace() # The number of hidden layers the network will have. @@ -118,6 +118,7 @@ def get_hyperparameter_search_space( default_value=num_units.default_value, log=num_units.log) n_units_hp = get_hyperparameter(n_units_search_space, UniformIntegerHyperparameter) + cs.add_hyperparameter(n_units_hp) if i > int(min_mlp_layers): diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py index 8ee3ed19b..cd8a07525 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py @@ -113,12 +113,14 @@ def get_hyperparameter_search_space( default_value=True, ), multi_branch_choice: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="mb_choice", - value_range=('None', 'shake-shake', 'shake-drop'), + value_range=('None', 'shake-shake', + 'shake-drop'), default_value='shake-drop', ), num_units: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="num_units", value_range=(10, 1024), default_value=200, + log=True ), activation: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="activation", value_range=tuple(_activations.keys()), diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/ShapedMLPBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/ShapedMLPBackbone.py index 46574642c..194f018aa 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/ShapedMLPBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/ShapedMLPBackbone.py @@ -96,11 +96,11 @@ def get_hyperparameter_search_space( max_units: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="max_units", value_range=(10, 1024), default_value=200, - ), + log=True), output_dim: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="output_dim", value_range=(10, 1024), default_value=200, - ), + log=True), mlp_shape: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="mlp_shape", value_range=('funnel', 'long_funnel', 'diamond', 'hexagon', @@ -114,7 +114,6 @@ def get_hyperparameter_search_space( ), ) -> ConfigurationSpace: - cs = ConfigurationSpace() # The number of groups that will compose the resnet. That is, diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py index 59cd45d5d..217253f91 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py @@ -98,6 +98,7 @@ def get_hyperparameter_search_space( # type: ignore[override] output_dim: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="output_dim", value_range=(10, 1024), default_value=200, + log=True ), num_groups: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="num_groups", value_range=(1, 15), @@ -116,12 +117,15 @@ def get_hyperparameter_search_space( # type: ignore[override] default_value=True, ), multi_branch_choice: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="mb_choice", - value_range=('None', 'shake-shake', 'shake-drop'), + value_range=('None', 'shake-shake', + 'shake-drop'), default_value='shake-drop', ), max_units: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="max_units", value_range=(10, 1024), - default_value=200), + default_value=200, + log=True + ), activation: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="activation", value_range=tuple(_activations.keys()), default_value=list(_activations.keys())[0]), @@ -154,6 +158,7 @@ def get_hyperparameter_search_space( # type: ignore[override] use_dropout = get_hyperparameter(use_dropout, CategoricalHyperparameter) max_dropout = get_hyperparameter(max_dropout, UniformFloatHyperparameter) + cs.add_hyperparameters([use_dropout, max_dropout]) cs.add_condition(CS.EqualsCondition(max_dropout, use_dropout, True)) use_sc = get_hyperparameter(use_skip_connection, CategoricalHyperparameter) diff --git a/autoPyTorch/pipeline/components/setup/optimizer/AdamOptimizer.py b/autoPyTorch/pipeline/components/setup/optimizer/AdamOptimizer.py index 2fef66aac..ab722940e 100644 --- a/autoPyTorch/pipeline/components/setup/optimizer/AdamOptimizer.py +++ b/autoPyTorch/pipeline/components/setup/optimizer/AdamOptimizer.py @@ -93,12 +93,13 @@ def get_hyperparameter_search_space( value_range=(0.9, 0.9999), default_value=0.9), use_weight_decay: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_weight_decay", - value_range=(True, False), - default_value=True, - ), + value_range=(True, False), + default_value=True, + ), weight_decay: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="weight_decay", - value_range=(0.0, 0.1), - default_value=0.0), + value_range=(1E-7, 0.1), + default_value=1E-4, + log=True), ) -> ConfigurationSpace: cs = ConfigurationSpace() diff --git a/autoPyTorch/pipeline/components/setup/optimizer/AdamWOptimizer.py b/autoPyTorch/pipeline/components/setup/optimizer/AdamWOptimizer.py index f7df85756..4ac43bc87 100644 --- a/autoPyTorch/pipeline/components/setup/optimizer/AdamWOptimizer.py +++ b/autoPyTorch/pipeline/components/setup/optimizer/AdamWOptimizer.py @@ -97,8 +97,9 @@ def get_hyperparameter_search_space( default_value=True, ), weight_decay: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="weight_decay", - value_range=(0.0, 0.1), - default_value=0.0), + value_range=(1E-7, 0.1), + default_value=1E-4, + log=True), ) -> ConfigurationSpace: cs = ConfigurationSpace() diff --git a/autoPyTorch/pipeline/components/setup/optimizer/RMSpropOptimizer.py b/autoPyTorch/pipeline/components/setup/optimizer/RMSpropOptimizer.py index d1dc6f077..a718ff1bd 100644 --- a/autoPyTorch/pipeline/components/setup/optimizer/RMSpropOptimizer.py +++ b/autoPyTorch/pipeline/components/setup/optimizer/RMSpropOptimizer.py @@ -97,8 +97,9 @@ def get_hyperparameter_search_space( default_value=True, ), weight_decay: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="weight_decay", - value_range=(0.0, 0.1), - default_value=0.0), + value_range=(1E-7, 0.1), + default_value=1E-4, + log=True), momentum: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="momentum", value_range=(0.0, 0.99), default_value=0.0), @@ -109,7 +110,6 @@ def get_hyperparameter_search_space( add_hyperparameter(cs, lr, UniformFloatHyperparameter) add_hyperparameter(cs, alpha, UniformFloatHyperparameter) add_hyperparameter(cs, momentum, UniformFloatHyperparameter) - weight_decay = get_hyperparameter(weight_decay, UniformFloatHyperparameter) use_weight_decay = get_hyperparameter(use_weight_decay, CategoricalHyperparameter) cs.add_hyperparameters([use_weight_decay, weight_decay]) diff --git a/autoPyTorch/pipeline/components/setup/optimizer/SGDOptimizer.py b/autoPyTorch/pipeline/components/setup/optimizer/SGDOptimizer.py index 492bdf97e..9b240f970 100644 --- a/autoPyTorch/pipeline/components/setup/optimizer/SGDOptimizer.py +++ b/autoPyTorch/pipeline/components/setup/optimizer/SGDOptimizer.py @@ -88,8 +88,9 @@ def get_hyperparameter_search_space( default_value=True, ), weight_decay: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="weight_decay", - value_range=(0.0, 0.1), - default_value=0.0), + value_range=(1E-7, 0.1), + default_value=1E-4, + log=True), momentum: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="momentum", value_range=(0.0, 0.99), default_value=0.0), diff --git a/autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py b/autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py index 483ac98d4..0036d4040 100644 --- a/autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py +++ b/autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py @@ -264,10 +264,12 @@ def get_hyperparameter_search_space( dataset_properties: Optional[Dict[str, BaseDatasetPropertiesType]] = None, batch_size: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="batch_size", value_range=(32, 320), - default_value=64) + default_value=64, + log=True) ) -> ConfigurationSpace: cs = ConfigurationSpace() add_hyperparameter(cs, batch_size, UniformIntegerHyperparameter) + return cs def __str__(self) -> str: diff --git a/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py b/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py index 166c85481..e8344844d 100644 --- a/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py @@ -1,5 +1,5 @@ -import typing from copy import deepcopy +from typing import Any, Callable, Dict, Optional, Tuple, Union from ConfigSpace.conditions import EqualsCondition from ConfigSpace.configuration_space import ConfigurationSpace @@ -17,6 +17,7 @@ from autoPyTorch.constants import CLASSIFICATION_TASKS, STRING_TO_TASK_TYPES from autoPyTorch.pipeline.components.training.trainer.base_trainer import BaseTrainerComponent from autoPyTorch.pipeline.components.training.trainer.utils import Lookahead +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter class AdversarialTrainer(BaseTrainerComponent): @@ -24,12 +25,12 @@ def __init__( self, epsilon: float, weighted_loss: bool = False, - random_state: typing.Optional[np.random.RandomState] = None, + random_state: Optional[np.random.RandomState] = None, use_stochastic_weight_averaging: bool = False, use_snapshot_ensemble: bool = False, se_lastk: int = 3, use_lookahead_optimizer: bool = True, - **lookahead_config: typing.Any + **lookahead_config: Any ): """ This class handles the training of a network for a single given epoch. @@ -48,7 +49,7 @@ def __init__( self.epsilon = epsilon def data_preparation(self, X: np.ndarray, y: np.ndarray, - ) -> typing.Tuple[typing.Tuple[np.ndarray, np.ndarray], typing.Dict[str, np.ndarray]]: + ) -> Tuple[Tuple[np.ndarray, np.ndarray], Dict[str, np.ndarray]]: """Generate adversarial examples from the original inputs. Args: @@ -63,7 +64,7 @@ def data_preparation(self, X: np.ndarray, y: np.ndarray, return (X, X_adversarial), {'y_a': y} def criterion_preparation(self, y_a: np.ndarray, y_b: np.ndarray = None, lam: float = 1.0 - ) -> typing.Callable: + ) -> Callable: # Initial implementation, consider the adversarial loss and the normal network loss # equally. return lambda criterion, pred, adversarial_pred: 0.5 * criterion(pred, y_a) + \ @@ -142,8 +143,8 @@ def fgsm_attack( return adv_data @staticmethod - def get_properties(dataset_properties: typing.Optional[typing.Dict[str, typing.Any]] = None - ) -> typing.Dict[str, typing.Union[str, bool]]: + def get_properties(dataset_properties: Optional[Dict[str, Any]] = None + ) -> Dict[str, Union[str, bool]]: return { 'shortname': 'AdversarialTrainer', @@ -155,53 +156,67 @@ def get_properties(dataset_properties: typing.Optional[typing.Dict[str, typing.A @staticmethod def get_hyperparameter_search_space( - dataset_properties: typing.Optional[typing.Dict] = None, - weighted_loss: typing.Tuple[typing.Tuple, bool] = ((True, False), True), - use_stochastic_weight_averaging: typing.Tuple[typing.Tuple, bool] = ((True, False), True), - use_snapshot_ensemble: typing.Tuple[typing.Tuple, bool] = ((True, False), True), - se_lastk: typing.Tuple[typing.Tuple, int] = ((3,), 3), - use_lookahead_optimizer: typing.Tuple[typing.Tuple, bool] = ((True, False), True), - la_steps: typing.Tuple[typing.Tuple, int, bool] = ((5, 10), 6, False), - la_alpha: typing.Tuple[typing.Tuple, float, bool] = ((0.5, 0.8), 0.6, False), - epsilon: typing.Tuple[typing.Tuple[float, float], float] = ((0.05, 0.2), 0.2), + dataset_properties: Optional[Dict] = None, + weighted_loss: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="weighted_loss", + value_range=(True, False), + default_value=True), + la_steps: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="la_steps", + value_range=(5, 10), + default_value=6, + log=False), + la_alpha: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="la_alpha", + value_range=(0.5, 0.8), + default_value=0.6, + log=False), + use_lookahead_optimizer: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="use_lookahead_optimizer", + value_range=(True, False), + default_value=True), + use_stochastic_weight_averaging: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="use_stochastic_weight_averaging", + value_range=(True, False), + default_value=True), + use_snapshot_ensemble: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="use_snapshot_ensemble", + value_range=(True, False), + default_value=True), + se_lastk: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="se_lastk", + value_range=(3,), + default_value=3), + epsilon: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="epsilon", + value_range=(0.05, 0.2), + default_value=0.2), ) -> ConfigurationSpace: - epsilon = UniformFloatHyperparameter( - "epsilon", epsilon[0][0], epsilon[0][1], default_value=epsilon[1]) - weighted_loss = CategoricalHyperparameter("weighted_loss", choices=weighted_loss[0], - default_value=weighted_loss[1]) - - use_swa = CategoricalHyperparameter("use_stochastic_weight_averaging", - choices=use_stochastic_weight_averaging[0], - default_value=use_stochastic_weight_averaging[1]) - use_se = CategoricalHyperparameter("use_snapshot_ensemble", - choices=use_snapshot_ensemble[0], - default_value=use_snapshot_ensemble[1]) - - # Note, this is not easy to be considered as a hyperparameter. - # When used with cyclic learning rates, it depends on the number - # of restarts. - se_lastk = Constant('se_lastk', se_lastk[1]) - - use_lookahead_optimizer = CategoricalHyperparameter("use_lookahead_optimizer", - choices=use_lookahead_optimizer[0], - default_value=use_lookahead_optimizer[1]) - - config_space = Lookahead.get_hyperparameter_search_space(la_steps=la_steps, - la_alpha=la_alpha) - parent_hyperparameter = {'parent': use_lookahead_optimizer, 'value': True} - cs = ConfigurationSpace() - cs.add_hyperparameters([use_swa, use_se, se_lastk, use_lookahead_optimizer]) + + add_hyperparameter(cs, epsilon, UniformFloatHyperparameter) + + get_hyperparameter(se_lastk, Constant) + add_hyperparameter(cs, use_stochastic_weight_averaging, CategoricalHyperparameter) + use_snapshot_ensemble = get_hyperparameter(use_snapshot_ensemble, CategoricalHyperparameter) + se_lastk = get_hyperparameter(se_lastk, Constant) + cs.add_hyperparameters([use_snapshot_ensemble, se_lastk]) + cond = EqualsCondition(se_lastk, use_snapshot_ensemble, True) + cs.add_condition(cond) + + use_lookahead_optimizer = get_hyperparameter(use_lookahead_optimizer, CategoricalHyperparameter) + cs.add_hyperparameter(use_lookahead_optimizer) + la_config_space = Lookahead.get_hyperparameter_search_space(la_steps=la_steps, + la_alpha=la_alpha) + parent_hyperparameter = {'parent': use_lookahead_optimizer, 'value': True} cs.add_configuration_space( Lookahead.__name__, - config_space, + la_config_space, parent_hyperparameter=parent_hyperparameter ) - cond = EqualsCondition(se_lastk, use_se, True) - cs.add_condition(cond) - cs.add_hyperparameters([epsilon]) if dataset_properties is not None: if STRING_TO_TASK_TYPES[dataset_properties['task_type']] in CLASSIFICATION_TASKS: - cs.add_hyperparameters([weighted_loss]) + add_hyperparameter(cs, weighted_loss, CategoricalHyperparameter) + return cs diff --git a/autoPyTorch/pipeline/components/training/trainer/base_trainer.py b/autoPyTorch/pipeline/components/training/trainer/base_trainer.py index 207fbf0dd..1feef1525 100644 --- a/autoPyTorch/pipeline/components/training/trainer/base_trainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/base_trainer.py @@ -32,7 +32,7 @@ from autoPyTorch.pipeline.components.training.trainer.utils import Lookahead from autoPyTorch.pipeline.components.training.metrics.utils import calculate_score from autoPyTorch.pipeline.components.training.trainer.utils import Lookahead, swa_average_function -from autoPyTorch.utils.common import FitRequirement +from autoPyTorch.utils.common import FitRequirement, HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter from autoPyTorch.utils.implementations import get_loss_weight_strategy @@ -592,49 +592,61 @@ def criterion_preparation(self, y_a: torch.Tensor, y_b: torch.Tensor = None, lam raise NotImplementedError() @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, - weighted_loss: Tuple[Tuple, bool] = ((True, False), True), - use_stochastic_weight_averaging: Tuple[Tuple, bool] = ((True, False), True), - use_snapshot_ensemble: Tuple[Tuple, bool] = ((True, False), True), - se_lastk: Tuple[Tuple, int] = ((3,), 3), - use_lookahead_optimizer: Tuple[Tuple, bool] = ((True, False), True), - la_steps: Tuple[Tuple, int, bool] = ((5, 10), 6, False), - la_alpha: Tuple[Tuple, float, bool] = ((0.5, 0.8), 0.6, False), - ) -> ConfigurationSpace: - weighted_loss = CategoricalHyperparameter("weighted_loss", choices=weighted_loss[0], - default_value=weighted_loss[1]) - use_swa = CategoricalHyperparameter("use_stochastic_weight_averaging", - choices=use_stochastic_weight_averaging[0], - default_value=use_stochastic_weight_averaging[1]) - use_se = CategoricalHyperparameter("use_snapshot_ensemble", - choices=use_snapshot_ensemble[0], - default_value=use_snapshot_ensemble[1]) - - # Note, this is not easy to be considered as a hyperparameter. - # When used with cyclic learning rates, it depends on the number - # of restarts. - se_lastk = Constant('se_lastk', se_lastk[1]) - - use_lookahead_optimizer = CategoricalHyperparameter("use_lookahead_optimizer", - choices=use_lookahead_optimizer[0], - default_value=use_lookahead_optimizer[1]) - - config_space = Lookahead.get_hyperparameter_search_space(la_steps=la_steps, - la_alpha=la_alpha) - parent_hyperparameter = {'parent': use_lookahead_optimizer, 'value': True} - + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict] = None, + weighted_loss: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="weighted_loss", + value_range=(True, False), + default_value=True), + la_steps: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="la_steps", + value_range=(5, 10), + default_value=6, + log=False), + la_alpha: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="la_alpha", + value_range=(0.5, 0.8), + default_value=0.6, + log=False), + use_lookahead_optimizer: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="use_lookahead_optimizer", + value_range=(True, False), + default_value=True), + use_stochastic_weight_averaging: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="use_stochastic_weight_averaging", + value_range=(True, False), + default_value=True), + use_snapshot_ensemble: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="use_snapshot_ensemble", + value_range=(True, False), + default_value=True), + se_lastk: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="se_lastk", + value_range=(3,), + default_value=3), + ) -> ConfigurationSpace: cs = ConfigurationSpace() - cs.add_hyperparameters([use_swa, use_se, se_lastk, use_lookahead_optimizer]) + + add_hyperparameter(cs, use_stochastic_weight_averaging, CategoricalHyperparameter) + use_snapshot_ensemble = get_hyperparameter(use_snapshot_ensemble, CategoricalHyperparameter) + se_lastk = get_hyperparameter(se_lastk, Constant) + cs.add_hyperparameters([use_snapshot_ensemble, se_lastk]) + cond = EqualsCondition(se_lastk, use_snapshot_ensemble, True) + cs.add_condition(cond) + + use_lookahead_optimizer = get_hyperparameter(use_lookahead_optimizer, CategoricalHyperparameter) + cs.add_hyperparameter(use_lookahead_optimizer) + la_config_space = Lookahead.get_hyperparameter_search_space(la_steps=la_steps, + la_alpha=la_alpha) + parent_hyperparameter = {'parent': use_lookahead_optimizer, 'value': True} cs.add_configuration_space( Lookahead.__name__, - config_space, + la_config_space, parent_hyperparameter=parent_hyperparameter ) - cond = EqualsCondition(se_lastk, use_se, True) - cs.add_condition(cond) if dataset_properties is not None: if STRING_TO_TASK_TYPES[dataset_properties['task_type']] in CLASSIFICATION_TASKS: - cs.add_hyperparameters([weighted_loss]) + add_hyperparameter(cs, weighted_loss, CategoricalHyperparameter) return cs diff --git a/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py b/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py index e5ef2ee1d..74cc9f935 100644 --- a/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py +++ b/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py @@ -58,14 +58,6 @@ def get_hyperparameter_search_space( hyperparameter="weighted_loss", value_range=(True, False), default_value=True), - patch_ratio: HyperparameterSearchSpace = HyperparameterSearchSpace( - hyperparameter="patch_ratio", - value_range=(0, 1), - default_value=0.2), - cutout_prob: HyperparameterSearchSpace = HyperparameterSearchSpace( - hyperparameter="cutout_prob", - value_range=(0, 1), - default_value=0.2), la_steps: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="la_steps", value_range=(5, 10), @@ -92,20 +84,29 @@ def get_hyperparameter_search_space( hyperparameter="se_lastk", value_range=(3, ), default_value=3), + patch_ratio: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="patch_ratio", + value_range=(0, 1), + default_value=0.2), + cutout_prob: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="cutout_prob", + value_range=(0, 1), + default_value=0.2), ) -> ConfigurationSpace: cs = ConfigurationSpace() add_hyperparameter(cs, patch_ratio, UniformFloatHyperparameter) add_hyperparameter(cs, cutout_prob, UniformFloatHyperparameter) - add_hyperparameter(cs, se_lastk, Constant) add_hyperparameter(cs, use_stochastic_weight_averaging, CategoricalHyperparameter) - use_snapshot_ensemble = get_hyperparameter(cs, use_snapshot_ensemble, CategoricalHyperparameter) - cs.add_hyperparameter(use_snapshot_ensemble) + use_snapshot_ensemble = get_hyperparameter(use_snapshot_ensemble, CategoricalHyperparameter) + se_lastk = get_hyperparameter(se_lastk, Constant) + cs.add_hyperparameters([use_snapshot_ensemble, se_lastk]) cond = EqualsCondition(se_lastk, use_snapshot_ensemble, True) cs.add_condition(cond) - add_hyperparameter(cs, use_lookahead_optimizer, CategoricalHyperparameter) + use_lookahead_optimizer = get_hyperparameter(use_lookahead_optimizer, CategoricalHyperparameter) + cs.add_hyperparameter(use_lookahead_optimizer) la_config_space = Lookahead.get_hyperparameter_search_space(la_steps=la_steps, la_alpha=la_alpha) parent_hyperparameter = {'parent': use_lookahead_optimizer, 'value': True} diff --git a/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py b/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py index 5b28b756f..f85474495 100644 --- a/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py +++ b/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py @@ -55,9 +55,6 @@ def get_hyperparameter_search_space( weighted_loss: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="weighted_loss", value_range=(True, False), default_value=True), - alpha: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="alpha", - value_range=(0, 1), - default_value=0.2), la_steps: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="la_steps", value_range=(5, 10), @@ -84,18 +81,23 @@ def get_hyperparameter_search_space( hyperparameter="se_lastk", value_range=(3,), default_value=3), + alpha: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="alpha", + value_range=(0, 1), + default_value=0.2), ) -> ConfigurationSpace: cs = ConfigurationSpace() add_hyperparameter(cs, alpha, UniformFloatHyperparameter) - add_hyperparameter(cs, se_lastk, Constant) add_hyperparameter(cs, use_stochastic_weight_averaging, CategoricalHyperparameter) - use_snapshot_ensemble = get_hyperparameter(cs, use_snapshot_ensemble, CategoricalHyperparameter) - cs.add_hyperparameter(use_snapshot_ensemble) + use_snapshot_ensemble = get_hyperparameter(use_snapshot_ensemble, CategoricalHyperparameter) + se_lastk = get_hyperparameter(se_lastk, Constant) + cs.add_hyperparameters([use_snapshot_ensemble, se_lastk]) cond = EqualsCondition(se_lastk, use_snapshot_ensemble, True) cs.add_condition(cond) - add_hyperparameter(cs, use_lookahead_optimizer, CategoricalHyperparameter) + use_lookahead_optimizer = get_hyperparameter(use_lookahead_optimizer, CategoricalHyperparameter) + cs.add_hyperparameter(use_lookahead_optimizer) la_config_space = Lookahead.get_hyperparameter_search_space(la_steps=la_steps, la_alpha=la_alpha) parent_hyperparameter = {'parent': use_lookahead_optimizer, 'value': True} diff --git a/test/conftest.py b/test/conftest.py index 2cf976d7a..622879d3c 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -310,6 +310,7 @@ def get_fit_dictionary(X, y, validator, backend): 'use_tensorboard_logger': True, 'metrics_during_training': True, 'split_id': 0, + 'use_pynisher': False, 'backend': backend, 'logger_port': logging.handlers.DEFAULT_TCP_LOGGING_PORT, } From 933989546801615b2847db3fec6a777c33f218b5 Mon Sep 17 00:00:00 2001 From: ArlindKadra Date: Mon, 12 Apr 2021 19:37:26 +0200 Subject: [PATCH 14/50] Adding constant clause in the testing module for hyperparameter ranges, removing use_pynisher since it is not used anymore, hack for the moment to bypass illegal hyperparameter ranges for conditions Fixing flake8 fail flake8 fix flake8 fix --- autoPyTorch/pipeline/base_pipeline.py | 1 - .../setup/network_backbone/ResNetBackbone.py | 7 ++++--- .../network_backbone/ShapedResNetBackbone.py | 7 ++++--- .../training/trainer/AdversarialTrainer.py | 2 -- .../training/trainer/MixUpTrainer.py | 1 + .../components/training/trainer/__init__.py | 21 ++++++------------- .../training/trainer/cutout_utils.py | 2 +- test/conftest.py | 1 - .../test_tabular_classification.py | 4 ++++ test/test_pipeline/test_tabular_regression.py | 2 +- 10 files changed, 21 insertions(+), 27 deletions(-) diff --git a/autoPyTorch/pipeline/base_pipeline.py b/autoPyTorch/pipeline/base_pipeline.py index 52b5f7579..6c6116a73 100644 --- a/autoPyTorch/pipeline/base_pipeline.py +++ b/autoPyTorch/pipeline/base_pipeline.py @@ -606,6 +606,5 @@ def get_default_pipeline_options() -> Dict[str, Any]: 'torch_num_threads': 1, 'early_stopping': 10, 'use_tensorboard_logger': True, - 'use_pynisher': False, 'metrics_during_training': True } diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py index cd8a07525..96e888581 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py @@ -112,9 +112,10 @@ def get_hyperparameter_search_space( value_range=(True, False), default_value=True, ), - multi_branch_choice: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="mb_choice", - value_range=('None', 'shake-shake', - 'shake-drop'), + multi_branch_choice: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="multi_branch_choice", + value_range=('shake-drop', + 'shake-shake', + 'None'), default_value='shake-drop', ), num_units: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="num_units", diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py index 217253f91..59135475f 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py @@ -116,9 +116,10 @@ def get_hyperparameter_search_space( # type: ignore[override] value_range=(True, False), default_value=True, ), - multi_branch_choice: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="mb_choice", - value_range=('None', 'shake-shake', - 'shake-drop'), + multi_branch_choice: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="multi_branch_choice", + value_range=('shake-drop', + 'shake-shake', + 'None'), default_value='shake-drop', ), max_units: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="max_units", diff --git a/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py b/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py index e8344844d..a51789f1e 100644 --- a/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py @@ -195,8 +195,6 @@ def get_hyperparameter_search_space( cs = ConfigurationSpace() add_hyperparameter(cs, epsilon, UniformFloatHyperparameter) - - get_hyperparameter(se_lastk, Constant) add_hyperparameter(cs, use_stochastic_weight_averaging, CategoricalHyperparameter) use_snapshot_ensemble = get_hyperparameter(use_snapshot_ensemble, CategoricalHyperparameter) se_lastk = get_hyperparameter(se_lastk, Constant) diff --git a/autoPyTorch/pipeline/components/training/trainer/MixUpTrainer.py b/autoPyTorch/pipeline/components/training/trainer/MixUpTrainer.py index b0f9cf696..2bd015b46 100644 --- a/autoPyTorch/pipeline/components/training/trainer/MixUpTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/MixUpTrainer.py @@ -1,6 +1,7 @@ from typing import Any, Dict, Optional, Tuple, Union import numpy as np + import torch from autoPyTorch.pipeline.components.training.trainer.base_trainer import BaseTrainerComponent diff --git a/autoPyTorch/pipeline/components/training/trainer/__init__.py b/autoPyTorch/pipeline/components/training/trainer/__init__.py index 84f6dacf5..81521e5cc 100755 --- a/autoPyTorch/pipeline/components/training/trainer/__init__.py +++ b/autoPyTorch/pipeline/components/training/trainer/__init__.py @@ -34,7 +34,7 @@ RunSummary, ) from autoPyTorch.pipeline.components.training.trainer.utils import Lookahead, update_model_state_dict_from_swa -from autoPyTorch.utils.common import FitRequirement, get_device_from_fit_dictionary +from autoPyTorch.utils.common import FitRequirement, HyperparameterSearchSpace, get_device_from_fit_dictionary from autoPyTorch.utils.logging_ import get_named_client_logger trainer_directory = os.path.split(__file__)[0] @@ -209,9 +209,9 @@ def get_hyperparameter_search_space( if default_ in available_trainers: default = default_ break - updates = self._get_search_space_updates() + updates: Dict[str, HyperparameterSearchSpace] = self._get_search_space_updates() if '__choice__' in updates.keys(): - choice_hyperparameter = updates['__choice__'] + choice_hyperparameter: HyperparameterSearchSpace = updates['__choice__'] if not set(choice_hyperparameter.value_range).issubset(available_trainers): raise ValueError("Expected given update for {} to have " "choices in {} got {}".format(self.__class__.__name__, @@ -284,19 +284,10 @@ def fit(self, X: Dict[str, Any], y: Any = None, **kwargs: Any) -> autoPyTorchCom ) # Add snapshots to base network to enable # predicting with snapshot ensemble - self.choice = cast(autoPyTorchComponent, self.choice) + self.choice: autoPyTorchComponent = cast(autoPyTorchComponent, self.choice) if self.choice.use_snapshot_ensemble: X['network_snapshots'].extend(self.choice.model_snapshots) - if X['use_pynisher']: - # Normally the X[network] is a pointer to the object, so at the - # end, when we train using X, the pipeline network is updated for free - # If we do multiprocessing (because of pynisher) we have to update - # X[network] manually. we do so in a way that every pipeline component - # can see this new network -- via an update, not overwrite of the pointer - state_dict = state_dict.result - X['network'].load_state_dict(state_dict) - # TODO: when have the optimizer code, the pynisher object might have failed # We should process this function as Failure if so trough fit_function.exit_status return self.choice @@ -680,7 +671,7 @@ def __str__(self) -> str: string = str(self.run_summary) return string - def _get_search_space_updates(self, prefix: Optional[str] = None) -> Dict[str, Tuple]: + def _get_search_space_updates(self, prefix: Optional[str] = None) -> Dict[str, HyperparameterSearchSpace]: """Get the search space updates with the given prefix Keyword Arguments: @@ -691,7 +682,7 @@ def _get_search_space_updates(self, prefix: Optional[str] = None) -> Dict[str, T """ updates = super()._get_search_space_updates(prefix=prefix) - result: Dict[str, Tuple] = dict() + result: Dict[str, HyperparameterSearchSpace] = dict() # iterate over all search space updates of this node and filter the ones out, that have the given prefix for key in updates.keys(): diff --git a/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py b/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py index 74cc9f935..8fd2a40cf 100644 --- a/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py +++ b/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py @@ -82,7 +82,7 @@ def get_hyperparameter_search_space( default_value=True), se_lastk: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="se_lastk", - value_range=(3, ), + value_range=(3,), default_value=3), patch_ratio: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="patch_ratio", diff --git a/test/conftest.py b/test/conftest.py index 622879d3c..2cf976d7a 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -310,7 +310,6 @@ def get_fit_dictionary(X, y, validator, backend): 'use_tensorboard_logger': True, 'metrics_during_training': True, 'split_id': 0, - 'use_pynisher': False, 'backend': backend, 'logger_port': logging.handlers.DEFAULT_TCP_LOGGING_PORT, } diff --git a/test/test_pipeline/test_tabular_classification.py b/test/test_pipeline/test_tabular_classification.py index 3569be2ad..cf852ae04 100644 --- a/test/test_pipeline/test_tabular_classification.py +++ b/test/test_pipeline/test_tabular_classification.py @@ -6,6 +6,7 @@ from ConfigSpace.configuration_space import Configuration from ConfigSpace.hyperparameters import ( CategoricalHyperparameter, + Constant, UniformFloatHyperparameter, UniformIntegerHyperparameter, ) @@ -330,6 +331,9 @@ def test_set_range_search_space_updates(self, fit_dictionary_tabular): if isinstance(hyperparameter, CategoricalHyperparameter): value_range = (hyperparameter.choices[0],) default_value = hyperparameter.choices[0] + elif isinstance(hyperparameter, Constant): + value_range = (hyperparameter.value,) + default_value = hyperparameter.value else: value_range = (0, 1) default_value = 1 diff --git a/test/test_pipeline/test_tabular_regression.py b/test/test_pipeline/test_tabular_regression.py index bc35ac796..f04d33f98 100644 --- a/test/test_pipeline/test_tabular_regression.py +++ b/test/test_pipeline/test_tabular_regression.py @@ -165,7 +165,7 @@ def test_remove_key_check_requirements(self, fit_dictionary_tabular): pipeline = TabularRegressionPipeline( dataset_properties=fit_dictionary_tabular['dataset_properties'], exclude={'trainer': ['AdversarialTrainer']}) - for key in ['num_run', 'device', 'split_id', 'use_pynisher', 'torch_num_threads', 'dataset_properties']: + for key in ['num_run', 'device', 'split_id', 'torch_num_threads', 'dataset_properties']: fit_dictionary_tabular_copy = fit_dictionary_tabular.copy() fit_dictionary_tabular_copy.pop(key) with pytest.raises(ValueError, match=r"To fit .+?, expected fit dictionary to have"): From 126f7d42b61e1d0d268ccb90a701e5993edb6332 Mon Sep 17 00:00:00 2001 From: ArlindKadra Date: Mon, 12 Apr 2021 23:13:52 +0200 Subject: [PATCH 15/50] Updating implementation for tabular regression Addressing the comment from Ravin Adding checks for the possible different value range of multibranch_choice Removing comment since it is not relevant anymore Removing unecessary code that transforms to cpu, while it is on cpu --- .../components/setup/network/base_network.py | 4 ++-- .../setup/network_backbone/ResNetBackbone.py | 15 ++++++++++----- .../network_backbone/ShapedResNetBackbone.py | 17 +++++++++++------ .../components/training/trainer/__init__.py | 2 -- test/test_pipeline/test_tabular_regression.py | 4 ++++ 5 files changed, 27 insertions(+), 15 deletions(-) diff --git a/autoPyTorch/pipeline/components/setup/network/base_network.py b/autoPyTorch/pipeline/components/setup/network/base_network.py index cb981e131..8b75ab66a 100644 --- a/autoPyTorch/pipeline/components/setup/network/base_network.py +++ b/autoPyTorch/pipeline/components/setup/network/base_network.py @@ -116,7 +116,7 @@ def predict(self, loader: torch.utils.data.DataLoader) -> torch.Tensor: """ if len(self.network_snapshots) == 0: assert self.network is not None - return self._predict(network=self.network, loader=loader).cpu().numpy() + return self._predict(network=self.network, loader=loader).numpy() else: # if there are network snapshots, # take average of predictions of all snapshots @@ -125,7 +125,7 @@ def predict(self, loader: torch.utils.data.DataLoader) -> torch.Tensor: for network in self.network_snapshots: Y_snapshot_preds.append(self._predict(network, loader)) Y_snapshot_preds_tensor = torch.stack(Y_snapshot_preds) - return Y_snapshot_preds_tensor.mean(dim=0).cpu().numpy() + return Y_snapshot_preds_tensor.mean(dim=0).numpy() def _predict(self, network: torch.nn.Module, loader: torch.utils.data.DataLoader) -> torch.Tensor: network.to(self.device) diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py index 96e888581..56857da8e 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py @@ -168,13 +168,18 @@ def get_hyperparameter_search_space( cs.add_hyperparameters([use_dropout]) use_sc = get_hyperparameter(use_skip_connection, CategoricalHyperparameter) + shake_drop_prob_flag = False + if 'shake-drop' in multi_branch_choice.value_range: + shake_drop_prob_flag = True mb_choice = get_hyperparameter(multi_branch_choice, CategoricalHyperparameter) - shake_drop_prob = get_hyperparameter(max_shake_drop_probability, UniformFloatHyperparameter) - cs.add_hyperparameters([use_sc, mb_choice, shake_drop_prob]) + + cs.add_hyperparameters([use_sc, mb_choice]) cs.add_condition(CS.EqualsCondition(mb_choice, use_sc, True)) - # TODO check if shake_drop is as an option in mb_choice - # Incomplete work - cs.add_condition(CS.EqualsCondition(shake_drop_prob, mb_choice, "shake-drop")) + + if shake_drop_prob_flag: + shake_drop_prob = get_hyperparameter(max_shake_drop_probability, UniformFloatHyperparameter) + cs.add_hyperparameter(shake_drop_prob) + cs.add_condition(CS.EqualsCondition(shake_drop_prob, mb_choice, "shake-drop")) # It is the upper bound of the nr of groups, # since the configuration will actually be sampled. diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py index 59135475f..6a92fb172 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py @@ -161,14 +161,19 @@ def get_hyperparameter_search_space( # type: ignore[override] max_dropout = get_hyperparameter(max_dropout, UniformFloatHyperparameter) cs.add_hyperparameters([use_dropout, max_dropout]) cs.add_condition(CS.EqualsCondition(max_dropout, use_dropout, True)) - use_sc = get_hyperparameter(use_skip_connection, CategoricalHyperparameter) + + shake_drop_prob_flag = False + if 'shake-drop' in multi_branch_choice.value_range: + shake_drop_prob_flag = True mb_choice = get_hyperparameter(multi_branch_choice, CategoricalHyperparameter) - shake_drop_prob = get_hyperparameter(max_shake_drop_probability, UniformFloatHyperparameter) - cs.add_hyperparameters([use_sc, mb_choice, shake_drop_prob]) + + cs.add_hyperparameters([use_sc, mb_choice]) cs.add_condition(CS.EqualsCondition(mb_choice, use_sc, True)) - # TODO check if shake_drop is as an option in mb_choice - # Incomplete work - cs.add_condition(CS.EqualsCondition(shake_drop_prob, mb_choice, "shake-drop")) + + if shake_drop_prob_flag: + shake_drop_prob = get_hyperparameter(max_shake_drop_probability, UniformFloatHyperparameter) + cs.add_hyperparameter(shake_drop_prob) + cs.add_condition(CS.EqualsCondition(shake_drop_prob, mb_choice, "shake-drop")) return cs diff --git a/autoPyTorch/pipeline/components/training/trainer/__init__.py b/autoPyTorch/pipeline/components/training/trainer/__init__.py index 81521e5cc..78ec8fe9c 100755 --- a/autoPyTorch/pipeline/components/training/trainer/__init__.py +++ b/autoPyTorch/pipeline/components/training/trainer/__init__.py @@ -288,8 +288,6 @@ def fit(self, X: Dict[str, Any], y: Any = None, **kwargs: Any) -> autoPyTorchCom if self.choice.use_snapshot_ensemble: X['network_snapshots'].extend(self.choice.model_snapshots) - # TODO: when have the optimizer code, the pynisher object might have failed - # We should process this function as Failure if so trough fit_function.exit_status return self.choice def prepare_trainer(self, X: Dict) -> None: diff --git a/test/test_pipeline/test_tabular_regression.py b/test/test_pipeline/test_tabular_regression.py index f04d33f98..c26be05b4 100644 --- a/test/test_pipeline/test_tabular_regression.py +++ b/test/test_pipeline/test_tabular_regression.py @@ -5,6 +5,7 @@ from ConfigSpace.hyperparameters import ( CategoricalHyperparameter, + Constant, UniformFloatHyperparameter, UniformIntegerHyperparameter, ) @@ -284,6 +285,9 @@ def test_set_range_search_space_updates(self, fit_dictionary_tabular): if isinstance(hyperparameter, CategoricalHyperparameter): value_range = (hyperparameter.choices[0],) default_value = hyperparameter.choices[0] + elif isinstance(hyperparameter, Constant): + value_range = (hyperparameter.value,) + default_value = hyperparameter.value else: value_range = (0, 1) default_value = 1 From f6f05ba674e15492ea6646533c068b685983949f Mon Sep 17 00:00:00 2001 From: ArlindKadra Date: Thu, 15 Apr 2021 12:03:48 +0200 Subject: [PATCH 16/50] Fixing buggy implementation of the network head with constant updates --- .../pipeline/components/setup/network_head/fully_connected.py | 1 - 1 file changed, 1 deletion(-) diff --git a/autoPyTorch/pipeline/components/setup/network_head/fully_connected.py b/autoPyTorch/pipeline/components/setup/network_head/fully_connected.py index 99762bbcf..3c01f75da 100644 --- a/autoPyTorch/pipeline/components/setup/network_head/fully_connected.py +++ b/autoPyTorch/pipeline/components/setup/network_head/fully_connected.py @@ -82,7 +82,6 @@ def get_hyperparameter_search_space( log=units_layer.log, ) num_units_hp = get_hyperparameter(num_units_search_space, UniformIntegerHyperparameter) - cs.add_hyperparameter(num_units_hp) if i >= min_num_layers and not num_layers_is_constant: # In the case of a constant, the max and min number of layers are the same. From 456e2617f030df312a5d4d3b9b482c5acd1053fd Mon Sep 17 00:00:00 2001 From: ArlindKadra Date: Tue, 20 Apr 2021 14:33:23 +0200 Subject: [PATCH 17/50] Updating implementation Temporary bug fix --- .../setup/network_backbone/MLPBackbone.py | 33 +++++++++++-------- .../setup/network_backbone/ResNetBackbone.py | 29 +++++++++------- .../network_backbone/ShapedMLPBackbone.py | 11 +++++-- .../network_backbone/ShapedResNetBackbone.py | 14 +++++--- 4 files changed, 54 insertions(+), 33 deletions(-) diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/MLPBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/MLPBackbone.py index c8777b032..625eddf55 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/MLPBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/MLPBackbone.py @@ -109,6 +109,10 @@ def get_hyperparameter_search_space( # We can have dropout in the network for # better generalization + dropout_flag = False + if any(use_dropout.value_range): + dropout_flag = True + use_dropout = get_hyperparameter(use_dropout, CategoricalHyperparameter) cs.add_hyperparameters([num_groups, use_dropout]) @@ -129,19 +133,20 @@ def get_hyperparameter_search_space( n_units_hp, num_groups, i - 1 ) ) - dropout_search_space = HyperparameterSearchSpace(hyperparameter='dropout_%d' % i, - value_range=dropout.value_range, - default_value=dropout.default_value, - log=dropout.log) - dropout_hp = get_hyperparameter(dropout_search_space, UniformFloatHyperparameter) - cs.add_hyperparameter(dropout_hp) - - dropout_condition_1 = CS.EqualsCondition(dropout_hp, use_dropout, True) - - if i > int(min_mlp_layers): - dropout_condition_2 = CS.GreaterThanCondition(dropout_hp, num_groups, i - 1) - cs.add_condition(CS.AndConjunction(dropout_condition_1, dropout_condition_2)) - else: - cs.add_condition(dropout_condition_1) + if dropout_flag: + dropout_search_space = HyperparameterSearchSpace(hyperparameter='dropout_%d' % i, + value_range=dropout.value_range, + default_value=dropout.default_value, + log=dropout.log) + dropout_hp = get_hyperparameter(dropout_search_space, UniformFloatHyperparameter) + cs.add_hyperparameter(dropout_hp) + + dropout_condition_1 = CS.EqualsCondition(dropout_hp, use_dropout, True) + + if i > int(min_mlp_layers): + dropout_condition_2 = CS.GreaterThanCondition(dropout_hp, num_groups, i - 1) + cs.add_condition(CS.AndConjunction(dropout_condition_1, dropout_condition_2)) + else: + cs.add_condition(dropout_condition_1) return cs diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py index 56857da8e..3db990109 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py @@ -164,6 +164,10 @@ def get_hyperparameter_search_space( # We can have dropout in the network for # better generalization + dropout_flag = False + if any(use_dropout.value_range): + dropout_flag = True + use_dropout = get_hyperparameter(use_dropout, CategoricalHyperparameter) cs.add_hyperparameters([use_dropout]) @@ -202,22 +206,23 @@ def get_hyperparameter_search_space( cs.add_condition(CS.GreaterThanCondition(n_units_hp, num_groups, i - 1)) cs.add_condition(CS.GreaterThanCondition(blocks_per_group_hp, num_groups, i - 1)) - dropout_search_space = HyperparameterSearchSpace(hyperparameter='dropout_%d' % i, - value_range=dropout.value_range, - default_value=dropout.default_value, - log=dropout.log) - dropout_hp = get_hyperparameter(dropout_search_space, UniformFloatHyperparameter) - cs.add_hyperparameter(dropout_hp) + if dropout_flag: + dropout_search_space = HyperparameterSearchSpace(hyperparameter='dropout_%d' % i, + value_range=dropout.value_range, + default_value=dropout.default_value, + log=dropout.log) + dropout_hp = get_hyperparameter(dropout_search_space, UniformFloatHyperparameter) + cs.add_hyperparameter(dropout_hp) - dropout_condition_1 = CS.EqualsCondition(dropout_hp, use_dropout, True) + dropout_condition_1 = CS.EqualsCondition(dropout_hp, use_dropout, True) - if i > 1: + if i > 1: - dropout_condition_2 = CS.GreaterThanCondition(dropout_hp, num_groups, i - 1) + dropout_condition_2 = CS.GreaterThanCondition(dropout_hp, num_groups, i - 1) - cs.add_condition(CS.AndConjunction(dropout_condition_1, dropout_condition_2)) - else: - cs.add_condition(dropout_condition_1) + cs.add_condition(CS.AndConjunction(dropout_condition_1, dropout_condition_2)) + else: + cs.add_condition(dropout_condition_1) return cs diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/ShapedMLPBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/ShapedMLPBackbone.py index 194f018aa..4e3a769a6 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/ShapedMLPBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/ShapedMLPBackbone.py @@ -127,10 +127,15 @@ def get_hyperparameter_search_space( # We can have dropout in the network for # better generalization + dropout_flag = False + if any(use_dropout.value_range): + dropout_flag = True use_dropout = get_hyperparameter(use_dropout, CategoricalHyperparameter) - max_dropout = get_hyperparameter(max_dropout, UniformFloatHyperparameter) + cs.add_hyperparameter(use_dropout) - cs.add_hyperparameters([use_dropout, max_dropout]) - cs.add_condition(CS.EqualsCondition(max_dropout, use_dropout, True)) + if dropout_flag: + max_dropout = get_hyperparameter(max_dropout, UniformFloatHyperparameter) + cs.add_hyperparameter(max_dropout) + cs.add_condition(CS.EqualsCondition(max_dropout, use_dropout, True)) return cs diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py index 6a92fb172..71a295210 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py @@ -157,12 +157,18 @@ def get_hyperparameter_search_space( # type: ignore[override] add_hyperparameter(cs, use_batch_norm, CategoricalHyperparameter) add_hyperparameter(cs, output_dim, UniformIntegerHyperparameter) + dropout_flag = False + if any(use_dropout.value_range): + dropout_flag = True use_dropout = get_hyperparameter(use_dropout, CategoricalHyperparameter) - max_dropout = get_hyperparameter(max_dropout, UniformFloatHyperparameter) - cs.add_hyperparameters([use_dropout, max_dropout]) - cs.add_condition(CS.EqualsCondition(max_dropout, use_dropout, True)) - use_sc = get_hyperparameter(use_skip_connection, CategoricalHyperparameter) + cs.add_hyperparameter(use_dropout) + + if dropout_flag: + max_dropout = get_hyperparameter(max_dropout, UniformFloatHyperparameter) + cs.add_hyperparameter(max_dropout) + cs.add_condition(CS.EqualsCondition(max_dropout, use_dropout, True)) + use_sc = get_hyperparameter(use_skip_connection, CategoricalHyperparameter) shake_drop_prob_flag = False if 'shake-drop' in multi_branch_choice.value_range: shake_drop_prob_flag = True From c7af6999e1a54b30e1398a3498433aeecfba184f Mon Sep 17 00:00:00 2001 From: ArlindKadra Date: Tue, 20 Apr 2021 15:27:54 +0200 Subject: [PATCH 18/50] Implementation fix for constant updates to skip connections. Multibranch choice was dependant on an illegal value --- .../setup/network_backbone/ResNetBackbone.py | 32 ++++++++++++------- .../network_backbone/ShapedResNetBackbone.py | 32 ++++++++++++------- 2 files changed, 40 insertions(+), 24 deletions(-) diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py index 3db990109..0e128d859 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py @@ -171,19 +171,27 @@ def get_hyperparameter_search_space( use_dropout = get_hyperparameter(use_dropout, CategoricalHyperparameter) cs.add_hyperparameters([use_dropout]) + skip_connection_flag = False + if any(use_skip_connection.value_range): + skip_connection_flag = True + use_sc = get_hyperparameter(use_skip_connection, CategoricalHyperparameter) - shake_drop_prob_flag = False - if 'shake-drop' in multi_branch_choice.value_range: - shake_drop_prob_flag = True - mb_choice = get_hyperparameter(multi_branch_choice, CategoricalHyperparameter) - - cs.add_hyperparameters([use_sc, mb_choice]) - cs.add_condition(CS.EqualsCondition(mb_choice, use_sc, True)) - - if shake_drop_prob_flag: - shake_drop_prob = get_hyperparameter(max_shake_drop_probability, UniformFloatHyperparameter) - cs.add_hyperparameter(shake_drop_prob) - cs.add_condition(CS.EqualsCondition(shake_drop_prob, mb_choice, "shake-drop")) + cs.add_hyperparameter(use_sc) + + if skip_connection_flag: + + shake_drop_prob_flag = False + if 'shake-drop' in multi_branch_choice.value_range: + shake_drop_prob_flag = True + + mb_choice = get_hyperparameter(multi_branch_choice, CategoricalHyperparameter) + cs.add_hyperparameter(mb_choice) + cs.add_condition(CS.EqualsCondition(mb_choice, use_sc, True)) + + if shake_drop_prob_flag: + shake_drop_prob = get_hyperparameter(max_shake_drop_probability, UniformFloatHyperparameter) + cs.add_hyperparameter(shake_drop_prob) + cs.add_condition(CS.EqualsCondition(shake_drop_prob, mb_choice, "shake-drop")) # It is the upper bound of the nr of groups, # since the configuration will actually be sampled. diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py index 71a295210..6383e6230 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py @@ -168,18 +168,26 @@ def get_hyperparameter_search_space( # type: ignore[override] cs.add_hyperparameter(max_dropout) cs.add_condition(CS.EqualsCondition(max_dropout, use_dropout, True)) + skip_connection_flag = False + if any(use_skip_connection.value_range): + skip_connection_flag = True + use_sc = get_hyperparameter(use_skip_connection, CategoricalHyperparameter) - shake_drop_prob_flag = False - if 'shake-drop' in multi_branch_choice.value_range: - shake_drop_prob_flag = True - mb_choice = get_hyperparameter(multi_branch_choice, CategoricalHyperparameter) - - cs.add_hyperparameters([use_sc, mb_choice]) - cs.add_condition(CS.EqualsCondition(mb_choice, use_sc, True)) - - if shake_drop_prob_flag: - shake_drop_prob = get_hyperparameter(max_shake_drop_probability, UniformFloatHyperparameter) - cs.add_hyperparameter(shake_drop_prob) - cs.add_condition(CS.EqualsCondition(shake_drop_prob, mb_choice, "shake-drop")) + cs.add_hyperparameter(use_sc) + + if skip_connection_flag: + + shake_drop_prob_flag = False + if 'shake-drop' in multi_branch_choice.value_range: + shake_drop_prob_flag = True + + mb_choice = get_hyperparameter(multi_branch_choice, CategoricalHyperparameter) + cs.add_hyperparameter(mb_choice) + cs.add_condition(CS.EqualsCondition(mb_choice, use_sc, True)) + + if shake_drop_prob_flag: + shake_drop_prob = get_hyperparameter(max_shake_drop_probability, UniformFloatHyperparameter) + cs.add_hyperparameter(shake_drop_prob) + cs.add_condition(CS.EqualsCondition(shake_drop_prob, mb_choice, "shake-drop")) return cs From 2102b088ac475acc25b8bbf735c6a1258b448d15 Mon Sep 17 00:00:00 2001 From: ArlindKadra Date: Tue, 20 Apr 2021 15:55:50 +0200 Subject: [PATCH 19/50] Fixing the implementation for weight decay in the case of fixed updates. The reason is the illegal value for weight decay when use weight decay is off --- .../setup/optimizer/AdamOptimizer.py | 24 ++++++++++++------- .../setup/optimizer/AdamWOptimizer.py | 24 ++++++++++++------- .../setup/optimizer/RMSpropOptimizer.py | 24 ++++++++++++------- .../setup/optimizer/SGDOptimizer.py | 24 ++++++++++++------- 4 files changed, 60 insertions(+), 36 deletions(-) diff --git a/autoPyTorch/pipeline/components/setup/optimizer/AdamOptimizer.py b/autoPyTorch/pipeline/components/setup/optimizer/AdamOptimizer.py index ab722940e..196848879 100644 --- a/autoPyTorch/pipeline/components/setup/optimizer/AdamOptimizer.py +++ b/autoPyTorch/pipeline/components/setup/optimizer/AdamOptimizer.py @@ -107,16 +107,22 @@ def get_hyperparameter_search_space( add_hyperparameter(cs, lr, UniformFloatHyperparameter) add_hyperparameter(cs, beta1, UniformFloatHyperparameter) add_hyperparameter(cs, beta2, UniformFloatHyperparameter) - weight_decay = get_hyperparameter(weight_decay, UniformFloatHyperparameter) - use_weight_decay = get_hyperparameter(use_weight_decay, CategoricalHyperparameter) - cs.add_hyperparameters([use_weight_decay, weight_decay]) + weight_decay_flag = False + if any(use_weight_decay.value_range): + weight_decay_flag = True - cs.add_condition( - CS.EqualsCondition( - weight_decay, - use_weight_decay, - True, + use_weight_decay = get_hyperparameter(use_weight_decay, CategoricalHyperparameter) + cs.add_hyperparameter(use_weight_decay) + + if weight_decay_flag: + weight_decay = get_hyperparameter(weight_decay, UniformFloatHyperparameter) + cs.add_hyperparameter(weight_decay) + cs.add_condition( + CS.EqualsCondition( + weight_decay, + use_weight_decay, + True, + ) ) - ) return cs diff --git a/autoPyTorch/pipeline/components/setup/optimizer/AdamWOptimizer.py b/autoPyTorch/pipeline/components/setup/optimizer/AdamWOptimizer.py index 4ac43bc87..3ae84a9e0 100644 --- a/autoPyTorch/pipeline/components/setup/optimizer/AdamWOptimizer.py +++ b/autoPyTorch/pipeline/components/setup/optimizer/AdamWOptimizer.py @@ -108,16 +108,22 @@ def get_hyperparameter_search_space( add_hyperparameter(cs, beta1, UniformFloatHyperparameter) add_hyperparameter(cs, beta2, UniformFloatHyperparameter) - weight_decay = get_hyperparameter(weight_decay, UniformFloatHyperparameter) - use_weight_decay = get_hyperparameter(use_weight_decay, CategoricalHyperparameter) - cs.add_hyperparameters([use_weight_decay, weight_decay]) + weight_decay_flag = False + if any(use_weight_decay.value_range): + weight_decay_flag = True - cs.add_condition( - CS.EqualsCondition( - weight_decay, - use_weight_decay, - True, + use_weight_decay = get_hyperparameter(use_weight_decay, CategoricalHyperparameter) + cs.add_hyperparameter(use_weight_decay) + + if weight_decay_flag: + weight_decay = get_hyperparameter(weight_decay, UniformFloatHyperparameter) + cs.add_hyperparameter(weight_decay) + cs.add_condition( + CS.EqualsCondition( + weight_decay, + use_weight_decay, + True, + ) ) - ) return cs diff --git a/autoPyTorch/pipeline/components/setup/optimizer/RMSpropOptimizer.py b/autoPyTorch/pipeline/components/setup/optimizer/RMSpropOptimizer.py index a718ff1bd..fc24323ad 100644 --- a/autoPyTorch/pipeline/components/setup/optimizer/RMSpropOptimizer.py +++ b/autoPyTorch/pipeline/components/setup/optimizer/RMSpropOptimizer.py @@ -110,16 +110,22 @@ def get_hyperparameter_search_space( add_hyperparameter(cs, lr, UniformFloatHyperparameter) add_hyperparameter(cs, alpha, UniformFloatHyperparameter) add_hyperparameter(cs, momentum, UniformFloatHyperparameter) - weight_decay = get_hyperparameter(weight_decay, UniformFloatHyperparameter) - use_weight_decay = get_hyperparameter(use_weight_decay, CategoricalHyperparameter) - cs.add_hyperparameters([use_weight_decay, weight_decay]) + weight_decay_flag = False + if any(use_weight_decay.value_range): + weight_decay_flag = True - cs.add_condition( - CS.EqualsCondition( - weight_decay, - use_weight_decay, - True, + use_weight_decay = get_hyperparameter(use_weight_decay, CategoricalHyperparameter) + cs.add_hyperparameter(use_weight_decay) + + if weight_decay_flag: + weight_decay = get_hyperparameter(weight_decay, UniformFloatHyperparameter) + cs.add_hyperparameter(weight_decay) + cs.add_condition( + CS.EqualsCondition( + weight_decay, + use_weight_decay, + True, + ) ) - ) return cs diff --git a/autoPyTorch/pipeline/components/setup/optimizer/SGDOptimizer.py b/autoPyTorch/pipeline/components/setup/optimizer/SGDOptimizer.py index 9b240f970..c8ed49c08 100644 --- a/autoPyTorch/pipeline/components/setup/optimizer/SGDOptimizer.py +++ b/autoPyTorch/pipeline/components/setup/optimizer/SGDOptimizer.py @@ -101,16 +101,22 @@ def get_hyperparameter_search_space( add_hyperparameter(cs, lr, UniformFloatHyperparameter) add_hyperparameter(cs, momentum, UniformFloatHyperparameter) - weight_decay = get_hyperparameter(weight_decay, UniformFloatHyperparameter) - use_weight_decay = get_hyperparameter(use_weight_decay, CategoricalHyperparameter) - cs.add_hyperparameters([use_weight_decay, weight_decay]) + weight_decay_flag = False + if any(use_weight_decay.value_range): + weight_decay_flag = True - cs.add_condition( - CS.EqualsCondition( - weight_decay, - use_weight_decay, - True, + use_weight_decay = get_hyperparameter(use_weight_decay, CategoricalHyperparameter) + cs.add_hyperparameter(use_weight_decay) + + if weight_decay_flag: + weight_decay = get_hyperparameter(weight_decay, UniformFloatHyperparameter) + cs.add_hyperparameter(weight_decay) + cs.add_condition( + CS.EqualsCondition( + weight_decay, + use_weight_decay, + True, + ) ) - ) return cs From 18da2bd6bf86f9b15bbb94f27e331e2302bea060 Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Wed, 21 Apr 2021 20:30:42 +0200 Subject: [PATCH 20/50] update setup.py --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 41f9f38f9..bd6fa5b11 100755 --- a/setup.py +++ b/setup.py @@ -72,6 +72,7 @@ "emcee", "scikit-optimize", "pyDOE", + "pytest-forked" ], "examples": [ "matplotlib", From 18bdabfa6ec0b4a303cebf9a8c20d1d91ab1179d Mon Sep 17 00:00:00 2001 From: ArlindKadra Date: Thu, 22 Apr 2021 14:06:00 +0200 Subject: [PATCH 21/50] Updating implementation of the reg cocktails so that it is compatible with fixed search space updates Turning off small preprocessing and also removing the TAE assertion for when the budget is larger than 100 Decreasing the dataset size for the fit pipeline since it is giving time out errors with its current implementation Updating implementation --- autoPyTorch/datasets/base_dataset.py | 3 +- .../training/trainer/AdversarialTrainer.py | 14 +++-- .../training/trainer/base_trainer.py | 16 ++++-- .../training/trainer/cutout_utils.py | 52 ++++++++++++------ .../training/trainer/mixup_utils.py | 54 +++++++++++++------ 5 files changed, 100 insertions(+), 39 deletions(-) diff --git a/autoPyTorch/datasets/base_dataset.py b/autoPyTorch/datasets/base_dataset.py index bd50cdbd6..8560462bc 100644 --- a/autoPyTorch/datasets/base_dataset.py +++ b/autoPyTorch/datasets/base_dataset.py @@ -165,7 +165,8 @@ def __init__( self.output_shape, self.output_type = _get_output_properties(self.train_tensors) # TODO: Look for a criteria to define small enough to preprocess - self.is_small_preprocess = True + # False for the regularization cocktails initially + self.is_small_preprocess = False # Make sure cross validation splits are created once self.cross_validators = CrossValFuncs.get_cross_validators(*CrossValTypes) diff --git a/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py b/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py index a51789f1e..3272d056b 100644 --- a/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py @@ -24,7 +24,7 @@ class AdversarialTrainer(BaseTrainerComponent): def __init__( self, epsilon: float, - weighted_loss: bool = False, + weighted_loss: int = 1, random_state: Optional[np.random.RandomState] = None, use_stochastic_weight_averaging: bool = False, use_snapshot_ensemble: bool = False, @@ -159,8 +159,8 @@ def get_hyperparameter_search_space( dataset_properties: Optional[Dict] = None, weighted_loss: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="weighted_loss", - value_range=(True, False), - default_value=True), + value_range=[1], + default_value=1), la_steps: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="la_steps", value_range=(5, 10), @@ -213,8 +213,16 @@ def get_hyperparameter_search_space( parent_hyperparameter=parent_hyperparameter ) + """ if dataset_properties is not None: if STRING_TO_TASK_TYPES[dataset_properties['task_type']] in CLASSIFICATION_TASKS: add_hyperparameter(cs, weighted_loss, CategoricalHyperparameter) + """ + # TODO, decouple the weighted loss from the trainer. Uncomment the code above and + # remove the code below. Also update the method signature, so the weighted loss + # is not a constant. + if dataset_properties is not None: + if STRING_TO_TASK_TYPES[dataset_properties['task_type']] in CLASSIFICATION_TASKS: + add_hyperparameter(cs, weighted_loss, Constant) return cs diff --git a/autoPyTorch/pipeline/components/training/trainer/base_trainer.py b/autoPyTorch/pipeline/components/training/trainer/base_trainer.py index 1feef1525..ceee79f90 100644 --- a/autoPyTorch/pipeline/components/training/trainer/base_trainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/base_trainer.py @@ -213,7 +213,7 @@ class BaseTrainerComponent(autoPyTorchTrainingComponent): """ Base class for training Args: - weighted_loss (bool, default=True): In case for classification, whether to weight + weighted_loss (int, default=1): In case for classification, whether to weight the loss function according to the distribution of classes in the target use_stochastic_weight_averaging (bool, default=True): whether to use stochastic weight averaging. Stochastic weight averaging is a simple average of @@ -228,7 +228,7 @@ class BaseTrainerComponent(autoPyTorchTrainingComponent): random_state: **lookahead_config: """ - def __init__(self, weighted_loss: bool = True, + def __init__(self, weighted_loss: int = 1, use_stochastic_weight_averaging: bool = True, use_snapshot_ensemble: bool = True, se_lastk: int = 3, @@ -596,8 +596,8 @@ def get_hyperparameter_search_space( dataset_properties: Optional[Dict] = None, weighted_loss: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="weighted_loss", - value_range=(True, False), - default_value=True), + value_range=[1], + default_value=1), la_steps: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="la_steps", value_range=(5, 10), @@ -645,8 +645,16 @@ def get_hyperparameter_search_space( parent_hyperparameter=parent_hyperparameter ) + """ if dataset_properties is not None: if STRING_TO_TASK_TYPES[dataset_properties['task_type']] in CLASSIFICATION_TASKS: add_hyperparameter(cs, weighted_loss, CategoricalHyperparameter) + """ + # TODO, decouple the weighted loss from the trainer. Uncomment the code above and + # remove the code below. Also update the method signature, so the weighted loss + # is not a constant. + if dataset_properties is not None: + if STRING_TO_TASK_TYPES[dataset_properties['task_type']] in CLASSIFICATION_TASKS: + add_hyperparameter(cs, weighted_loss, Constant) return cs diff --git a/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py b/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py index 8fd2a40cf..9e72dd671 100644 --- a/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py +++ b/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py @@ -18,7 +18,7 @@ class CutOut: def __init__(self, patch_ratio: float, cutout_prob: float, - weighted_loss: bool = False, + weighted_loss: int = 1, random_state: Optional[np.random.RandomState] = None, use_stochastic_weight_averaging: bool = False, use_snapshot_ensemble: bool = False, @@ -56,8 +56,9 @@ def get_hyperparameter_search_space( dataset_properties: Optional[Dict] = None, weighted_loss: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="weighted_loss", - value_range=(True, False), - default_value=True), + value_range=[1], + default_value=1 + ), la_steps: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="la_steps", value_range=(5, 10), @@ -99,25 +100,46 @@ def get_hyperparameter_search_space( add_hyperparameter(cs, patch_ratio, UniformFloatHyperparameter) add_hyperparameter(cs, cutout_prob, UniformFloatHyperparameter) add_hyperparameter(cs, use_stochastic_weight_averaging, CategoricalHyperparameter) + snapshot_ensemble_flag = False + if any(use_snapshot_ensemble.value_range): + snapshot_ensemble_flag = True + use_snapshot_ensemble = get_hyperparameter(use_snapshot_ensemble, CategoricalHyperparameter) - se_lastk = get_hyperparameter(se_lastk, Constant) - cs.add_hyperparameters([use_snapshot_ensemble, se_lastk]) - cond = EqualsCondition(se_lastk, use_snapshot_ensemble, True) - cs.add_condition(cond) + cs.add_hyperparameter(use_snapshot_ensemble) + + if snapshot_ensemble_flag: + se_lastk = get_hyperparameter(se_lastk, Constant) + cs.add_hyperparameter(se_lastk) + cond = EqualsCondition(se_lastk, use_snapshot_ensemble, True) + cs.add_condition(cond) + + lookahead_flag = False + if any(use_lookahead_optimizer.value_range): + lookahead_flag = True use_lookahead_optimizer = get_hyperparameter(use_lookahead_optimizer, CategoricalHyperparameter) cs.add_hyperparameter(use_lookahead_optimizer) - la_config_space = Lookahead.get_hyperparameter_search_space(la_steps=la_steps, - la_alpha=la_alpha) - parent_hyperparameter = {'parent': use_lookahead_optimizer, 'value': True} - cs.add_configuration_space( - Lookahead.__name__, - la_config_space, - parent_hyperparameter=parent_hyperparameter - ) + if lookahead_flag: + la_config_space = Lookahead.get_hyperparameter_search_space(la_steps=la_steps, + la_alpha=la_alpha) + parent_hyperparameter = {'parent': use_lookahead_optimizer, 'value': True} + cs.add_configuration_space( + Lookahead.__name__, + la_config_space, + parent_hyperparameter=parent_hyperparameter + ) + + """ if dataset_properties is not None: if STRING_TO_TASK_TYPES[dataset_properties['task_type']] in CLASSIFICATION_TASKS: add_hyperparameter(cs, weighted_loss, CategoricalHyperparameter) + """ + # TODO, decouple the weighted loss from the trainer. Uncomment the code above and + # remove the code below. Also update the method signature, so the weighted loss + # is not a constant. + if dataset_properties is not None: + if STRING_TO_TASK_TYPES[dataset_properties['task_type']] in CLASSIFICATION_TASKS: + add_hyperparameter(cs, weighted_loss, Constant) return cs diff --git a/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py b/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py index f85474495..4c1ed06f2 100644 --- a/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py +++ b/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py @@ -17,7 +17,7 @@ class MixUp: def __init__(self, alpha: float, - weighted_loss: bool = False, + weighted_loss: int = 1, random_state: Optional[np.random.RandomState] = None, use_stochastic_weight_averaging: bool = False, use_snapshot_ensemble: bool = False, @@ -52,9 +52,11 @@ def criterion_preparation(self, y_a: np.ndarray, y_b: np.ndarray = None, lam: fl @staticmethod def get_hyperparameter_search_space( dataset_properties: Optional[Dict] = None, - weighted_loss: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="weighted_loss", - value_range=(True, False), - default_value=True), + weighted_loss: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="weighted_loss", + value_range=[1], + default_value=1 + ), la_steps: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="la_steps", value_range=(5, 10), @@ -90,25 +92,45 @@ def get_hyperparameter_search_space( cs = ConfigurationSpace() add_hyperparameter(cs, alpha, UniformFloatHyperparameter) add_hyperparameter(cs, use_stochastic_weight_averaging, CategoricalHyperparameter) + snapshot_ensemble_flag = False + if any(use_snapshot_ensemble.value_range): + snapshot_ensemble_flag = True + use_snapshot_ensemble = get_hyperparameter(use_snapshot_ensemble, CategoricalHyperparameter) - se_lastk = get_hyperparameter(se_lastk, Constant) - cs.add_hyperparameters([use_snapshot_ensemble, se_lastk]) - cond = EqualsCondition(se_lastk, use_snapshot_ensemble, True) - cs.add_condition(cond) + cs.add_hyperparameter(use_snapshot_ensemble) + + if snapshot_ensemble_flag: + se_lastk = get_hyperparameter(se_lastk, Constant) + cs.add_hyperparameter(se_lastk) + cond = EqualsCondition(se_lastk, use_snapshot_ensemble, True) + cs.add_condition(cond) + + lookahead_flag = False + if any(use_lookahead_optimizer.value_range): + lookahead_flag = True use_lookahead_optimizer = get_hyperparameter(use_lookahead_optimizer, CategoricalHyperparameter) cs.add_hyperparameter(use_lookahead_optimizer) - la_config_space = Lookahead.get_hyperparameter_search_space(la_steps=la_steps, - la_alpha=la_alpha) - parent_hyperparameter = {'parent': use_lookahead_optimizer, 'value': True} - cs.add_configuration_space( - Lookahead.__name__, - la_config_space, - parent_hyperparameter=parent_hyperparameter - ) + if lookahead_flag: + la_config_space = Lookahead.get_hyperparameter_search_space(la_steps=la_steps, + la_alpha=la_alpha) + parent_hyperparameter = {'parent': use_lookahead_optimizer, 'value': True} + cs.add_configuration_space( + Lookahead.__name__, + la_config_space, + parent_hyperparameter=parent_hyperparameter + ) + """ if dataset_properties is not None: if STRING_TO_TASK_TYPES[dataset_properties['task_type']] in CLASSIFICATION_TASKS: add_hyperparameter(cs, weighted_loss, CategoricalHyperparameter) + """ + # TODO, decouple the weighted loss from the trainer. Uncomment the code above and + # remove the code below. Also update the method signature, so the weighted loss + # is not a constant. + if dataset_properties is not None: + if STRING_TO_TASK_TYPES[dataset_properties['task_type']] in CLASSIFICATION_TASKS: + add_hyperparameter(cs, weighted_loss, Constant) return cs From 0c2c604f0aa974ff4b48728b643cc245bbb34456 Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Fri, 30 Apr 2021 14:37:16 +0200 Subject: [PATCH 22/50] Create fit evaluator, no resampling strategy and fix bug for test statistics Fix mypy and flake Fix check for X_test while making test data loader fix bug in lookahead hyperparameters where lookahead was repeated for the hyperparameter name Make passing tests in api easier Fix bug in trainer weighted loss code for regression --- autoPyTorch/api/base_task.py | 1 + autoPyTorch/api/tabular_classification.py | 8 + autoPyTorch/api/tabular_regression.py | 8 + autoPyTorch/datasets/base_dataset.py | 34 +++ autoPyTorch/datasets/resampling_strategy.py | 1 + autoPyTorch/evaluation/abstract_evaluator.py | 9 +- autoPyTorch/evaluation/train_evaluator.py | 9 + .../training/data_loader/base_data_loader.py | 6 +- .../training/trainer/AdversarialTrainer.py | 2 +- .../training/trainer/StandardTrainer.py | 2 +- .../components/training/trainer/__init__.py | 22 +- .../training/trainer/base_trainer.py | 4 +- .../training/trainer/cutout_utils.py | 2 +- .../training/trainer/mixup_utils.py | 2 +- test/test_evaluation/test_fit_evaluator.py | 205 ++++++++++++++++++ .../test_tabular_classification.py | 15 ++ 16 files changed, 316 insertions(+), 14 deletions(-) create mode 100644 test/test_evaluation/test_fit_evaluator.py diff --git a/autoPyTorch/api/base_task.py b/autoPyTorch/api/base_task.py index c5468eae7..07d27c6d4 100644 --- a/autoPyTorch/api/base_task.py +++ b/autoPyTorch/api/base_task.py @@ -1578,6 +1578,7 @@ def fit_pipeline( (BaseDataset): Dataset created from the given tensors """ + self.dataset_name = dataset.dataset_name if dataset is None: if ( diff --git a/autoPyTorch/api/tabular_classification.py b/autoPyTorch/api/tabular_classification.py index facb59f99..766662fcd 100644 --- a/autoPyTorch/api/tabular_classification.py +++ b/autoPyTorch/api/tabular_classification.py @@ -92,9 +92,17 @@ def __init__( output_directory: Optional[str] = None, delete_tmp_folder_after_terminate: bool = True, delete_output_folder_after_terminate: bool = True, +<<<<<<< HEAD include_components: Optional[Dict[str, Any]] = None, exclude_components: Optional[Dict[str, Any]] = None, resampling_strategy: ResamplingStrategies = HoldoutValTypes.holdout_validation, +======= + include_components: Optional[Dict] = None, + exclude_components: Optional[Dict] = None, + resampling_strategy: Union[CrossValTypes, + HoldoutValTypes, + NoResamplingStrategyTypes] = HoldoutValTypes.holdout_validation, +>>>>>>> Create fit evaluator, no resampling strategy and fix bug for test statistics resampling_strategy_args: Optional[Dict[str, Any]] = None, backend: Optional[Backend] = None, search_space_updates: Optional[HyperparameterSearchSpaceUpdates] = None diff --git a/autoPyTorch/api/tabular_regression.py b/autoPyTorch/api/tabular_regression.py index e0c1e4eac..cec05aca2 100644 --- a/autoPyTorch/api/tabular_regression.py +++ b/autoPyTorch/api/tabular_regression.py @@ -93,9 +93,17 @@ def __init__( output_directory: Optional[str] = None, delete_tmp_folder_after_terminate: bool = True, delete_output_folder_after_terminate: bool = True, +<<<<<<< HEAD include_components: Optional[Dict[str, Any]] = None, exclude_components: Optional[Dict[str, Any]] = None, resampling_strategy: ResamplingStrategies = HoldoutValTypes.holdout_validation, +======= + include_components: Optional[Dict] = None, + exclude_components: Optional[Dict] = None, + resampling_strategy:Union[CrossValTypes, + HoldoutValTypes, + NoResamplingStrategyTypes] = HoldoutValTypes.holdout_validation, +>>>>>>> Create fit evaluator, no resampling strategy and fix bug for test statistics resampling_strategy_args: Optional[Dict[str, Any]] = None, backend: Optional[Backend] = None, search_space_updates: Optional[HyperparameterSearchSpaceUpdates] = None diff --git a/autoPyTorch/datasets/base_dataset.py b/autoPyTorch/datasets/base_dataset.py index 8560462bc..224517b0d 100644 --- a/autoPyTorch/datasets/base_dataset.py +++ b/autoPyTorch/datasets/base_dataset.py @@ -112,7 +112,13 @@ def __init__( dataset_name: Optional[str] = None, val_tensors: Optional[BaseDatasetInputType] = None, test_tensors: Optional[BaseDatasetInputType] = None, +<<<<<<< HEAD resampling_strategy: ResamplingStrategies = HoldoutValTypes.holdout_validation, +======= + resampling_strategy: Union[CrossValTypes, + HoldoutValTypes, + NoResamplingStrategyTypes] = HoldoutValTypes.holdout_validation, +>>>>>>> Create fit evaluator, no resampling strategy and fix bug for test statistics resampling_strategy_args: Optional[Dict[str, Any]] = None, shuffle: Optional[bool] = True, seed: Optional[int] = 42, @@ -129,7 +135,12 @@ def __init__( validation data test_tensors (An optional tuple of objects that have a __len__ and a __getitem__ attribute): test data +<<<<<<< HEAD resampling_strategy (RESAMPLING_STRATEGIES: default=HoldoutValTypes.holdout_validation): +======= + resampling_strategy (Union[CrossValTypes, HoldoutValTypes, NoResamplingStrategyTypes]), + (default=HoldoutValTypes.holdout_validation): +>>>>>>> Create fit evaluator, no resampling strategy and fix bug for test statistics strategy to split the training data. resampling_strategy_args (Optional[Dict[str, Any]]): arguments required for the chosen resampling strategy. If None, uses @@ -151,10 +162,17 @@ def __init__( if not hasattr(train_tensors[0], 'shape'): type_check(train_tensors, val_tensors) self.train_tensors, self.val_tensors, self.test_tensors = train_tensors, val_tensors, test_tensors +<<<<<<< HEAD self.cross_validators: Dict[str, CrossValFunc] = {} self.holdout_validators: Dict[str, HoldOutFunc] = {} self.no_resampling_validators: Dict[str, NoResamplingFunc] = {} self.random_state = np.random.RandomState(seed=seed) +======= + self.cross_validators: Dict[str, CROSS_VAL_FN] = {} + self.holdout_validators: Dict[str, HOLDOUT_FN] = {} + self.no_resampling_validators: Dict[str, NO_RESAMPLING_FN] = {} + self.rng = np.random.RandomState(seed=seed) +>>>>>>> Fix mypy and flake self.shuffle = shuffle self.resampling_strategy = resampling_strategy self.resampling_strategy_args = resampling_strategy_args @@ -171,7 +189,11 @@ def __init__( # Make sure cross validation splits are created once self.cross_validators = CrossValFuncs.get_cross_validators(*CrossValTypes) self.holdout_validators = HoldOutFuncs.get_holdout_validators(*HoldoutValTypes) +<<<<<<< HEAD self.no_resampling_validators = NoResamplingFuncs.get_no_resampling_validators(*NoResamplingStrategyTypes) +======= + self.no_resampling_validators = get_no_resampling_validators(*NoResamplingStrategyTypes) +>>>>>>> Create fit evaluator, no resampling strategy and fix bug for test statistics self.splits = self.get_splits_from_resampling_strategy() @@ -272,8 +294,12 @@ def get_splits_from_resampling_strategy(self) -> List[Tuple[List[int], Optional[ ) ) elif isinstance(self.resampling_strategy, NoResamplingStrategyTypes): +<<<<<<< HEAD splits.append((self.no_resampling_validators[self.resampling_strategy.name](self.random_state, self._get_indices()), None)) +======= + splits.append((self.no_resampling_validators[self.resampling_strategy.name](self._get_indices()), None)) +>>>>>>> Create fit evaluator, no resampling strategy and fix bug for test statistics else: raise ValueError(f"Unsupported resampling strategy={self.resampling_strategy}") return splits @@ -345,7 +371,11 @@ def create_holdout_val_split( self.random_state, val_share, self._get_indices(), **kwargs) return train, val +<<<<<<< HEAD def get_dataset(self, split_id: int, train: bool) -> Dataset: +======= + def get_dataset_for_training(self, split_id: int, train: bool) -> Dataset: +>>>>>>> Create fit evaluator, no resampling strategy and fix bug for test statistics """ The above split methods employ the Subset to internally subsample the whole dataset. @@ -360,6 +390,7 @@ def get_dataset(self, split_id: int, train: bool) -> Dataset: Dataset: the reduced dataset to be used for testing """ # Subset creates a dataset. Splits is a (train_indices, test_indices) tuple +<<<<<<< HEAD if split_id >= len(self.splits): # old version: split_id > len(self.splits) raise IndexError(f"self.splits index out of range, got split_id={split_id}" f" (>= num_splits={len(self.splits)})") @@ -368,6 +399,9 @@ def get_dataset(self, split_id: int, train: bool) -> Dataset: raise ValueError("Specified fold (or subset) does not exist") return TransformSubset(self, indices, train=train) +======= + return TransformSubset(self, self.splits[split_id][0], train=train) +>>>>>>> Create fit evaluator, no resampling strategy and fix bug for test statistics def replace_data(self, X_train: BaseDatasetInputType, X_test: Optional[BaseDatasetInputType]) -> 'BaseDataset': diff --git a/autoPyTorch/datasets/resampling_strategy.py b/autoPyTorch/datasets/resampling_strategy.py index 4f373bf24..a85207087 100644 --- a/autoPyTorch/datasets/resampling_strategy.py +++ b/autoPyTorch/datasets/resampling_strategy.py @@ -110,6 +110,7 @@ def is_stratified(self) -> bool: # TODO: replace it with another way ResamplingStrategies = Union[CrossValTypes, HoldoutValTypes, NoResamplingStrategyTypes] + DEFAULT_RESAMPLING_PARAMETERS: Dict[ ResamplingStrategies, Dict[str, Any] diff --git a/autoPyTorch/evaluation/abstract_evaluator.py b/autoPyTorch/evaluation/abstract_evaluator.py index d1bd3c43e..c45e4db3c 100644 --- a/autoPyTorch/evaluation/abstract_evaluator.py +++ b/autoPyTorch/evaluation/abstract_evaluator.py @@ -727,9 +727,9 @@ def _loss(self, y_true: np.ndarray, y_hat: np.ndarray, **metric_kwargs: Any) -> y_true, y_hat, self.task_type, metrics, **metric_kwargs) def finish_up(self, loss: Dict[str, float], train_loss: Dict[str, float], - opt_pred: np.ndarray, valid_pred: Optional[np.ndarray], - test_pred: Optional[np.ndarray], additional_run_info: Optional[Dict], - file_output: bool, status: StatusType, **metric_kwargs: Any + valid_pred: Optional[np.ndarray], test_pred: Optional[np.ndarray], + additional_run_info: Optional[Dict], file_output: bool, status: StatusType, + opt_pred: Optional[np.ndarray], ) -> Optional[Tuple[float, float, int, Dict]]: """This function does everything necessary after the fitting is done: @@ -773,6 +773,9 @@ def finish_up(self, loss: Dict[str, float], train_loss: Dict[str, float], Additional run information, like train/test loss """ + assert opt_pred is not None, "Cases where 'opt_pred' is None should be handled " \ + "specifically with special child classes" + self.duration = time.time() - self.starttime if file_output: diff --git a/autoPyTorch/evaluation/train_evaluator.py b/autoPyTorch/evaluation/train_evaluator.py index 142af6bcc..ec870cdb3 100644 --- a/autoPyTorch/evaluation/train_evaluator.py +++ b/autoPyTorch/evaluation/train_evaluator.py @@ -152,6 +152,11 @@ def __init__(self, backend: Backend, queue: Queue, pipeline_config=pipeline_config, search_space_updates=search_space_updates ) + assert isinstance(self.datamanager.resampling_strategy, (CrossValTypes, HoldoutValTypes)),\ + "This Evaluator is used for HPO Search. " \ + "Val Split is required for HPO search. " \ + "Expected 'self.resampling_strategy' in" \ + " '(CrossValTypes, HoldoutValTypes)' got {}".format(self.datamanager.resampling_strategy) if not isinstance(self.resampling_strategy, (CrossValTypes, HoldoutValTypes)): raise ValueError( @@ -408,7 +413,11 @@ def _predict(self, pipeline: BaseEstimator, # create closure for evaluating an algorithm +<<<<<<< HEAD def eval_train_function( +======= +def eval_function( +>>>>>>> Create fit evaluator, no resampling strategy and fix bug for test statistics backend: Backend, queue: Queue, metric: autoPyTorchMetric, diff --git a/autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py b/autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py index 0036d4040..769713680 100644 --- a/autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py +++ b/autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py @@ -59,15 +59,15 @@ def __init__(self, batch_size: int = 64, FitRequirement("Backend", (Backend,), user_defined=True, dataset_property=False), FitRequirement("is_small_preprocess", (bool,), user_defined=True, dataset_property=True)]) - def transform(self, X: Dict) -> Dict: + def transform(self, X: Dict[str, Any]) -> Dict[str, Any]: """The transform function calls the transform function of the underlying model and returns the transformed array. Args: - X (np.ndarray): input features + X (Dict[str, Any])): 'X' dictionary Returns: - np.ndarray: Transformed features + (Dict[str, Any]): the updated 'X' dictionary """ X.update({'train_data_loader': self.train_data_loader, 'val_data_loader': self.val_data_loader, diff --git a/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py b/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py index 3272d056b..d96e2117d 100644 --- a/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py @@ -24,7 +24,7 @@ class AdversarialTrainer(BaseTrainerComponent): def __init__( self, epsilon: float, - weighted_loss: int = 1, + weighted_loss: int = 0, random_state: Optional[np.random.RandomState] = None, use_stochastic_weight_averaging: bool = False, use_snapshot_ensemble: bool = False, diff --git a/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py b/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py index 825e1c034..4d6cf52a1 100644 --- a/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py @@ -13,7 +13,7 @@ class StandardTrainer(BaseTrainerComponent): - def __init__(self, weighted_loss: bool = False, + def __init__(self, weighted_loss: int = 0, use_stochastic_weight_averaging: bool = False, use_snapshot_ensemble: bool = False, se_lastk: int = 3, diff --git a/autoPyTorch/pipeline/components/training/trainer/__init__.py b/autoPyTorch/pipeline/components/training/trainer/__init__.py index 78ec8fe9c..b246cc0c2 100755 --- a/autoPyTorch/pipeline/components/training/trainer/__init__.py +++ b/autoPyTorch/pipeline/components/training/trainer/__init__.py @@ -396,7 +396,11 @@ def _fit(self, X: Dict[str, Any], y: Any = None, **kwargs: Any) -> 'TrainerChoic val_loss, val_metrics, test_loss, test_metrics = None, {}, None, {} if self.eval_valid_each_epoch(X): +<<<<<<< HEAD if X['val_data_loader']: +======= + if 'val_data_loader' in X and X['val_data_loader']: +>>>>>>> Create fit evaluator, no resampling strategy and fix bug for test statistics val_loss, val_metrics = self.choice.evaluate(X['val_data_loader'], epoch, writer) if 'test_data_loader' in X and X['test_data_loader']: test_loss, test_metrics = self.choice.evaluate(X['test_data_loader'], epoch, writer) @@ -450,10 +454,17 @@ def _fit(self, X: Dict[str, Any], y: Any = None, **kwargs: Any) -> 'TrainerChoic # wrap up -- add score if not evaluating every epoch if not self.eval_valid_each_epoch(X): +<<<<<<< HEAD if X['val_data_loader']: val_loss, val_metrics = self.choice.evaluate(X['val_data_loader'], epoch, writer) if 'test_data_loader' in X and X['val_data_loader']: test_loss, test_metrics = self.choice.evaluate(X['test_data_loader'], epoch, writer) +======= + if 'val_data_loader' in X and X['val_data_loader']: + val_loss, val_metrics = self.choice.evaluate(X['val_data_loader'], epoch, writer) + if 'test_data_loader' in X and X['test_data_loader']: + test_loss, test_metrics = self.choice.evaluate(X['test_data_loader']) +>>>>>>> Create fit evaluator, no resampling strategy and fix bug for test statistics self.run_summary.add_performance( epoch=epoch, start_time=start_time, @@ -684,8 +695,15 @@ def _get_search_space_updates(self, prefix: Optional[str] = None) -> Dict[str, H # iterate over all search space updates of this node and filter the ones out, that have the given prefix for key in updates.keys(): - if key.startswith(Lookahead.__name__): - result[key[len(Lookahead.__name__) + 1:]] = updates[key] + if Lookahead.__name__ in key: + # need to also remove lookahead from the hyperparameter name + new_update = HyperparameterSearchSpace( + updates[key].hyperparameter.replace('{}:'.format(Lookahead.__name__), ''), + value_range=updates[key].value_range, + default_value=updates[key].default_value, + log=updates[key].log + ) + result[key.replace('{}:'.format(Lookahead.__name__), '')] = new_update else: result[key] = updates[key] return result diff --git a/autoPyTorch/pipeline/components/training/trainer/base_trainer.py b/autoPyTorch/pipeline/components/training/trainer/base_trainer.py index ceee79f90..22535de55 100644 --- a/autoPyTorch/pipeline/components/training/trainer/base_trainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/base_trainer.py @@ -213,7 +213,7 @@ class BaseTrainerComponent(autoPyTorchTrainingComponent): """ Base class for training Args: - weighted_loss (int, default=1): In case for classification, whether to weight + weighted_loss (int, default=0): In case for classification, whether to weight the loss function according to the distribution of classes in the target use_stochastic_weight_averaging (bool, default=True): whether to use stochastic weight averaging. Stochastic weight averaging is a simple average of @@ -228,7 +228,7 @@ class BaseTrainerComponent(autoPyTorchTrainingComponent): random_state: **lookahead_config: """ - def __init__(self, weighted_loss: int = 1, + def __init__(self, weighted_loss: int = 0, use_stochastic_weight_averaging: bool = True, use_snapshot_ensemble: bool = True, se_lastk: int = 3, diff --git a/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py b/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py index 9e72dd671..aee666afa 100644 --- a/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py +++ b/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py @@ -18,7 +18,7 @@ class CutOut: def __init__(self, patch_ratio: float, cutout_prob: float, - weighted_loss: int = 1, + weighted_loss: int = 0, random_state: Optional[np.random.RandomState] = None, use_stochastic_weight_averaging: bool = False, use_snapshot_ensemble: bool = False, diff --git a/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py b/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py index 4c1ed06f2..7cd408c8a 100644 --- a/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py +++ b/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py @@ -17,7 +17,7 @@ class MixUp: def __init__(self, alpha: float, - weighted_loss: int = 1, + weighted_loss: int = 0, random_state: Optional[np.random.RandomState] = None, use_stochastic_weight_averaging: bool = False, use_snapshot_ensemble: bool = False, diff --git a/test/test_evaluation/test_fit_evaluator.py b/test/test_evaluation/test_fit_evaluator.py new file mode 100644 index 000000000..4e760a50c --- /dev/null +++ b/test/test_evaluation/test_fit_evaluator.py @@ -0,0 +1,205 @@ +import multiprocessing +import os +import queue +import shutil +import sys +import unittest +import unittest.mock + +from ConfigSpace import Configuration + +import numpy as np + +from sklearn.base import BaseEstimator + +from smac.tae import StatusType + +from autoPyTorch.datasets.resampling_strategy import NoResamplingStrategyTypes +from autoPyTorch.evaluation.fit_evaluator import FitEvaluator +from autoPyTorch.evaluation.utils import read_queue +from autoPyTorch.pipeline.base_pipeline import BasePipeline +from autoPyTorch.pipeline.components.training.metrics.metrics import accuracy +from autoPyTorch.utils import backend + +this_directory = os.path.dirname(__file__) +sys.path.append(this_directory) +from evaluation_util import ( # noqa (E402: module level import not at top of file) + BaseEvaluatorTest, + get_binary_classification_datamanager, + get_multiclass_classification_datamanager, + get_regression_datamanager, +) # noqa (E402: module level import not at top of file) + + +class BackendMock(object): + def load_datamanager(self): + return get_multiclass_classification_datamanager() + + +class Dummy(object): + def __init__(self): + self.name = 'dummy' + + +class DummyPipeline(BasePipeline): + def __init__(self): + mocked_estimator = unittest.mock.Mock(spec=BaseEstimator) + self.steps = [('MockStep', mocked_estimator)] + pass + + def predict_proba(self, X, batch_size=None): + return np.tile([0.6, 0.4], (len(X), 1)) + + def get_additional_run_info(self): + return {} + + +class TestFitEvaluator(BaseEvaluatorTest, unittest.TestCase): + _multiprocess_can_split_ = True + + def setUp(self): + """ + Creates a backend mock + """ + tmp_dir_name = self.id() + self.ev_path = os.path.join(this_directory, '.tmp_evaluations', tmp_dir_name) + if os.path.exists(self.ev_path): + shutil.rmtree(self.ev_path) + os.makedirs(self.ev_path, exist_ok=False) + dummy_model_files = [os.path.join(self.ev_path, str(n)) for n in range(100)] + dummy_pred_files = [os.path.join(self.ev_path, str(n)) for n in range(100, 200)] + dummy_cv_model_files = [os.path.join(self.ev_path, str(n)) for n in range(200, 300)] + backend_mock = unittest.mock.Mock() + backend_mock.get_model_dir.return_value = self.ev_path + backend_mock.get_cv_model_dir.return_value = self.ev_path + backend_mock.get_model_path.side_effect = dummy_model_files + backend_mock.get_cv_model_path.side_effect = dummy_cv_model_files + backend_mock.get_prediction_output_path.side_effect = dummy_pred_files + backend_mock.temporary_directory = self.ev_path + self.backend_mock = backend_mock + + self.tmp_dir = os.path.join(self.ev_path, 'tmp_dir') + self.output_dir = os.path.join(self.ev_path, 'out_dir') + + def tearDown(self): + if os.path.exists(self.ev_path): + shutil.rmtree(self.ev_path) + + @unittest.mock.patch('autoPyTorch.pipeline.tabular_classification.TabularClassificationPipeline') + def test_no_resampling(self, pipeline_mock): + # Binary iris, contains 69 train samples, 31 test samples + D = get_binary_classification_datamanager(NoResamplingStrategyTypes.no_resampling) + pipeline_mock.predict_proba.side_effect = \ + lambda X, batch_size=None: np.tile([0.6, 0.4], (len(X), 1)) + pipeline_mock.side_effect = lambda **kwargs: pipeline_mock + pipeline_mock.get_additional_run_info.return_value = None + + configuration = unittest.mock.Mock(spec=Configuration) + backend_api = backend.create(self.tmp_dir, self.output_dir) + backend_api.load_datamanager = lambda: D + queue_ = multiprocessing.Queue() + + evaluator = FitEvaluator(backend_api, queue_, configuration=configuration, metric=accuracy, budget=0) + evaluator.file_output = unittest.mock.Mock(spec=evaluator.file_output) + evaluator.file_output.return_value = (None, {}) + + evaluator.fit_predict_and_loss() + + rval = read_queue(evaluator.queue) + self.assertEqual(len(rval), 1) + result = rval[0]['loss'] + self.assertEqual(len(rval[0]), 3) + self.assertRaises(queue.Empty, evaluator.queue.get, timeout=1) + + self.assertEqual(evaluator.file_output.call_count, 1) + self.assertEqual(result, 0.5806451612903225) + self.assertEqual(pipeline_mock.fit.call_count, 1) + # 2 calls because of train and test set + self.assertEqual(pipeline_mock.predict_proba.call_count, 2) + self.assertEqual(evaluator.file_output.call_count, 1) + # Should be none as no val preds are mentioned + self.assertIsNone(evaluator.file_output.call_args[0][0]) + # Number of y_test_preds and Y_test should be the same + self.assertEqual(evaluator.file_output.call_args[0][2].shape[0], + D.test_tensors[1].shape[0]) + self.assertEqual(evaluator.pipeline.fit.call_count, 1) + + @unittest.mock.patch.object(FitEvaluator, '_loss') + def test_file_output(self, loss_mock): + + D = get_regression_datamanager(NoResamplingStrategyTypes.no_resampling) + D.name = 'test' + self.backend_mock.load_datamanager.return_value = D + configuration = unittest.mock.Mock(spec=Configuration) + queue_ = multiprocessing.Queue() + loss_mock.return_value = None + + evaluator = FitEvaluator(self.backend_mock, queue_, configuration=configuration, metric=accuracy, budget=0) + + self.backend_mock.get_model_dir.return_value = True + evaluator.pipeline = 'model' + evaluator.Y_optimization = D.train_tensors[1] + rval = evaluator.file_output( + D.train_tensors[1], + None, + D.test_tensors[1], + ) + + self.assertEqual(rval, (None, {})) + # These targets are not saved as Fit evaluator is not used to make an ensemble + self.assertEqual(self.backend_mock.save_targets_ensemble.call_count, 0) + self.assertEqual(self.backend_mock.save_numrun_to_dir.call_count, 1) + self.assertEqual(self.backend_mock.save_numrun_to_dir.call_args_list[-1][1].keys(), + {'seed', 'idx', 'budget', 'model', 'cv_model', + 'ensemble_predictions', 'valid_predictions', 'test_predictions'}) + self.assertIsNotNone(self.backend_mock.save_numrun_to_dir.call_args_list[-1][1]['model']) + self.assertIsNone(self.backend_mock.save_numrun_to_dir.call_args_list[-1][1]['cv_model']) + + # Check for not containing NaNs - that the models don't predict nonsense + # for unseen data + D.test_tensors[1][0] = np.NaN + rval = evaluator.file_output( + D.train_tensors[1], + None, + D.test_tensors[1], + ) + self.assertEqual( + rval, + ( + 1.0, + { + 'error': + 'Model predictions for test set contains NaNs.' + }, + ) + ) + + @unittest.mock.patch('autoPyTorch.pipeline.tabular_classification.TabularClassificationPipeline') + def test_predict_proba_binary_classification(self, mock): + D = get_binary_classification_datamanager(NoResamplingStrategyTypes.no_resampling) + self.backend_mock.load_datamanager.return_value = D + mock.predict_proba.side_effect = lambda y, batch_size=None: np.array( + [[0.1, 0.9]] * y.shape[0] + ) + mock.side_effect = lambda **kwargs: mock + + configuration = unittest.mock.Mock(spec=Configuration) + queue_ = multiprocessing.Queue() + + evaluator = FitEvaluator(self.backend_mock, queue_, configuration=configuration, metric=accuracy, budget=0) + + evaluator.fit_predict_and_loss() + Y_test_pred = self.backend_mock.save_numrun_to_dir.call_args_list[0][1][ + 'test_predictions'] + + for i in range(7): + self.assertEqual(0.9, Y_test_pred[i][1]) + + def test_get_results(self): + queue_ = multiprocessing.Queue() + for i in range(5): + queue_.put((i * 1, 1 - (i * 0.2), 0, "", StatusType.SUCCESS)) + result = read_queue(queue_) + self.assertEqual(len(result), 5) + self.assertEqual(result[0][0], 0) + self.assertAlmostEqual(result[0][1], 1.0) diff --git a/test/test_pipeline/test_tabular_classification.py b/test/test_pipeline/test_tabular_classification.py index cf852ae04..cc89be942 100644 --- a/test/test_pipeline/test_tabular_classification.py +++ b/test/test_pipeline/test_tabular_classification.py @@ -71,6 +71,8 @@ def test_pipeline_fit(self, fit_dictionary_tabular, exclude): """This test makes sure that the pipeline is able to fit given random combinations of hyperparameters across the pipeline""" + fit_dictionary_tabular['epochs'] = 5 + pipeline = TabularClassificationPipeline( dataset_properties=fit_dictionary_tabular['dataset_properties'], exclude=exclude) @@ -98,6 +100,9 @@ def test_pipeline_fit(self, fit_dictionary_tabular, exclude): def test_pipeline_predict(self, fit_dictionary_tabular, exclude): """This test makes sure that the pipeline is able to predict given a random configuration""" + + fit_dictionary_tabular['epochs'] = 5 + X = fit_dictionary_tabular['X_train'].copy() pipeline = TabularClassificationPipeline( dataset_properties=fit_dictionary_tabular['dataset_properties'], @@ -125,6 +130,9 @@ def test_pipeline_predict_proba(self, fit_dictionary_tabular, exclude): given random combinations of hyperparameters across the pipeline And then predict using predict probability """ + + fit_dictionary_tabular['epochs'] = 5 + X = fit_dictionary_tabular['X_train'].copy() pipeline = TabularClassificationPipeline( dataset_properties=fit_dictionary_tabular['dataset_properties'], @@ -158,6 +166,8 @@ def test_pipeline_transform(self, fit_dictionary_tabular, exclude): This code is added in light of components not properly added to the fit dicitonary """ + fit_dictionary_tabular['epochs'] = 5 + pipeline = TabularClassificationPipeline( dataset_properties=fit_dictionary_tabular['dataset_properties'], exclude=exclude) @@ -193,6 +203,8 @@ def test_default_configuration(self, fit_dictionary_tabular, is_small_preprocess """Makes sure that when no config is set, we can trust the default configuration from the space""" + fit_dictionary_tabular['epochs'] = 5 + fit_dictionary_tabular['is_small_preprocess'] = is_small_preprocess pipeline = TabularClassificationPipeline( @@ -205,6 +217,9 @@ def test_default_configuration(self, fit_dictionary_tabular, is_small_preprocess def test_remove_key_check_requirements(self, fit_dictionary_tabular): """Makes sure that when a key is removed from X, correct error is outputted""" + + fit_dictionary_tabular['epochs'] = 5 + pipeline = TabularClassificationPipeline( dataset_properties=fit_dictionary_tabular['dataset_properties']) for key in ['num_run', 'device', 'split_id', 'torch_num_threads', 'dataset_properties']: From 6d4790fa7efa616147c1ac05deb7d0ea4b09084b Mon Sep 17 00:00:00 2001 From: Ravin Kohli <13005107+ravinkohli@users.noreply.github.com> Date: Mon, 3 May 2021 13:46:43 +0200 Subject: [PATCH 23/50] Additional metrics during train (#194) * Added additional metrics to fit dictionary * Added in test also Fix mypy and flake after rebase, added random state to mixup and cutout and changs no resampling for new code fix bug in setup.py --- autoPyTorch/datasets/base_dataset.py | 36 ++----------------- autoPyTorch/datasets/resampling_strategy.py | 3 ++ .../training/trainer/cutout_utils.py | 9 ++++- .../training/trainer/mixup_utils.py | 9 ++++- test/test_api/test_api.py | 2 +- 5 files changed, 22 insertions(+), 37 deletions(-) diff --git a/autoPyTorch/datasets/base_dataset.py b/autoPyTorch/datasets/base_dataset.py index 224517b0d..755bf1e18 100644 --- a/autoPyTorch/datasets/base_dataset.py +++ b/autoPyTorch/datasets/base_dataset.py @@ -112,13 +112,7 @@ def __init__( dataset_name: Optional[str] = None, val_tensors: Optional[BaseDatasetInputType] = None, test_tensors: Optional[BaseDatasetInputType] = None, -<<<<<<< HEAD resampling_strategy: ResamplingStrategies = HoldoutValTypes.holdout_validation, -======= - resampling_strategy: Union[CrossValTypes, - HoldoutValTypes, - NoResamplingStrategyTypes] = HoldoutValTypes.holdout_validation, ->>>>>>> Create fit evaluator, no resampling strategy and fix bug for test statistics resampling_strategy_args: Optional[Dict[str, Any]] = None, shuffle: Optional[bool] = True, seed: Optional[int] = 42, @@ -135,12 +129,7 @@ def __init__( validation data test_tensors (An optional tuple of objects that have a __len__ and a __getitem__ attribute): test data -<<<<<<< HEAD resampling_strategy (RESAMPLING_STRATEGIES: default=HoldoutValTypes.holdout_validation): -======= - resampling_strategy (Union[CrossValTypes, HoldoutValTypes, NoResamplingStrategyTypes]), - (default=HoldoutValTypes.holdout_validation): ->>>>>>> Create fit evaluator, no resampling strategy and fix bug for test statistics strategy to split the training data. resampling_strategy_args (Optional[Dict[str, Any]]): arguments required for the chosen resampling strategy. If None, uses @@ -162,17 +151,11 @@ def __init__( if not hasattr(train_tensors[0], 'shape'): type_check(train_tensors, val_tensors) self.train_tensors, self.val_tensors, self.test_tensors = train_tensors, val_tensors, test_tensors -<<<<<<< HEAD self.cross_validators: Dict[str, CrossValFunc] = {} self.holdout_validators: Dict[str, HoldOutFunc] = {} self.no_resampling_validators: Dict[str, NoResamplingFunc] = {} self.random_state = np.random.RandomState(seed=seed) -======= - self.cross_validators: Dict[str, CROSS_VAL_FN] = {} - self.holdout_validators: Dict[str, HOLDOUT_FN] = {} - self.no_resampling_validators: Dict[str, NO_RESAMPLING_FN] = {} - self.rng = np.random.RandomState(seed=seed) ->>>>>>> Fix mypy and flake + self.no_resampling_validators: Dict[str, NoResamplingFunc] = {} self.shuffle = shuffle self.resampling_strategy = resampling_strategy self.resampling_strategy_args = resampling_strategy_args @@ -189,11 +172,8 @@ def __init__( # Make sure cross validation splits are created once self.cross_validators = CrossValFuncs.get_cross_validators(*CrossValTypes) self.holdout_validators = HoldOutFuncs.get_holdout_validators(*HoldoutValTypes) -<<<<<<< HEAD + self.no_resampling_validators = NoResamplingFuncs.get_no_resampling_validators(*NoResamplingStrategyTypes) -======= - self.no_resampling_validators = get_no_resampling_validators(*NoResamplingStrategyTypes) ->>>>>>> Create fit evaluator, no resampling strategy and fix bug for test statistics self.splits = self.get_splits_from_resampling_strategy() @@ -294,12 +274,8 @@ def get_splits_from_resampling_strategy(self) -> List[Tuple[List[int], Optional[ ) ) elif isinstance(self.resampling_strategy, NoResamplingStrategyTypes): -<<<<<<< HEAD splits.append((self.no_resampling_validators[self.resampling_strategy.name](self.random_state, self._get_indices()), None)) -======= - splits.append((self.no_resampling_validators[self.resampling_strategy.name](self._get_indices()), None)) ->>>>>>> Create fit evaluator, no resampling strategy and fix bug for test statistics else: raise ValueError(f"Unsupported resampling strategy={self.resampling_strategy}") return splits @@ -371,11 +347,7 @@ def create_holdout_val_split( self.random_state, val_share, self._get_indices(), **kwargs) return train, val -<<<<<<< HEAD def get_dataset(self, split_id: int, train: bool) -> Dataset: -======= - def get_dataset_for_training(self, split_id: int, train: bool) -> Dataset: ->>>>>>> Create fit evaluator, no resampling strategy and fix bug for test statistics """ The above split methods employ the Subset to internally subsample the whole dataset. @@ -390,7 +362,6 @@ def get_dataset_for_training(self, split_id: int, train: bool) -> Dataset: Dataset: the reduced dataset to be used for testing """ # Subset creates a dataset. Splits is a (train_indices, test_indices) tuple -<<<<<<< HEAD if split_id >= len(self.splits): # old version: split_id > len(self.splits) raise IndexError(f"self.splits index out of range, got split_id={split_id}" f" (>= num_splits={len(self.splits)})") @@ -399,9 +370,6 @@ def get_dataset_for_training(self, split_id: int, train: bool) -> Dataset: raise ValueError("Specified fold (or subset) does not exist") return TransformSubset(self, indices, train=train) -======= - return TransformSubset(self, self.splits[split_id][0], train=train) ->>>>>>> Create fit evaluator, no resampling strategy and fix bug for test statistics def replace_data(self, X_train: BaseDatasetInputType, X_test: Optional[BaseDatasetInputType]) -> 'BaseDataset': diff --git a/autoPyTorch/datasets/resampling_strategy.py b/autoPyTorch/datasets/resampling_strategy.py index a85207087..e2ac2736b 100644 --- a/autoPyTorch/datasets/resampling_strategy.py +++ b/autoPyTorch/datasets/resampling_strategy.py @@ -109,7 +109,10 @@ def is_stratified(self) -> bool: # TODO: replace it with another way ResamplingStrategies = Union[CrossValTypes, HoldoutValTypes, NoResamplingStrategyTypes] +<<<<<<< HEAD +======= +>>>>>>> Additional metrics during train (#194) DEFAULT_RESAMPLING_PARAMETERS: Dict[ ResamplingStrategies, diff --git a/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py b/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py index aee666afa..d073ea6cf 100644 --- a/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py +++ b/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py @@ -10,6 +10,8 @@ import numpy as np +from sklearn.utils import check_random_state + from autoPyTorch.constants import CLASSIFICATION_TASKS, STRING_TO_TASK_TYPES from autoPyTorch.pipeline.components.training.trainer.utils import Lookahead from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter @@ -35,7 +37,12 @@ def __init__(self, patch_ratio: float, """ self.use_stochastic_weight_averaging = use_stochastic_weight_averaging self.weighted_loss = weighted_loss - self.random_state = random_state + if random_state is None: + # A trainer components need a random state for + # sampling -- for example in MixUp training + self.random_state = check_random_state(1) + else: + self.random_state = random_state self.use_snapshot_ensemble = use_snapshot_ensemble self.se_lastk = se_lastk self.use_lookahead_optimizer = use_lookahead_optimizer diff --git a/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py b/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py index 7cd408c8a..ada279658 100644 --- a/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py +++ b/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py @@ -10,6 +10,8 @@ import numpy as np +from sklearn.utils import check_random_state + from autoPyTorch.constants import CLASSIFICATION_TASKS, STRING_TO_TASK_TYPES from autoPyTorch.pipeline.components.training.trainer.utils import Lookahead from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter @@ -34,7 +36,12 @@ def __init__(self, alpha: float, """ self.use_stochastic_weight_averaging = use_stochastic_weight_averaging self.weighted_loss = weighted_loss - self.random_state = random_state + if random_state is None: + # A trainer components need a random state for + # sampling -- for example in MixUp training + self.random_state = check_random_state(1) + else: + self.random_state = random_state self.use_snapshot_ensemble = use_snapshot_ensemble self.se_lastk = se_lastk self.use_lookahead_optimizer = use_lookahead_optimizer diff --git a/test/test_api/test_api.py b/test/test_api/test_api.py index 3c9bbd1a7..bf562dcde 100644 --- a/test/test_api/test_api.py +++ b/test/test_api/test_api.py @@ -41,7 +41,7 @@ from autoPyTorch.pipeline.components.setup.traditional_ml.traditional_learner import _traditional_learners from autoPyTorch.pipeline.components.training.metrics.metrics import accuracy -from test.test_api.api_utils import print_debug_information +from test.test_api.api_utils import print_debug_information # noqa E402 CV_NUM_SPLITS = 2 From 5168ba5c450ee3d9682f23882c112695a23b62b8 Mon Sep 17 00:00:00 2001 From: Arlind Kadra Date: Fri, 7 May 2021 14:06:06 +0200 Subject: [PATCH 24/50] Fixing issues with imbalanced datasets (#197) * adding missing method from base_feature_validator * First try at a fix, removing redundant code * Fix bug * Updating unit test typo, fixing bug where the data type was not checked because X was a numpy array at the time of checking * Fixing flake 8 failing * Bug fix, implementation update for imbalanced datasets and unit tests to check the implementation * flake8 fix * Bug fix * Making the conversion to dataframe in the unit tests consistent with what happens at the validator, so the types do not change * flake8 fix * Addressing Ravin's comments --- autoPyTorch/data/base_feature_validator.py | 14 ++++ autoPyTorch/data/tabular_feature_validator.py | 65 +++++++++------- test/test_data/test_feature_validator.py | 75 +++++++++++++++++-- test/test_data/test_validation.py | 1 - 4 files changed, 120 insertions(+), 35 deletions(-) diff --git a/autoPyTorch/data/base_feature_validator.py b/autoPyTorch/data/base_feature_validator.py index 2d09c474e..c7facd997 100644 --- a/autoPyTorch/data/base_feature_validator.py +++ b/autoPyTorch/data/base_feature_validator.py @@ -111,6 +111,20 @@ def _fit( """ raise NotImplementedError() + def _check_data( + self, + X: SUPPORTED_FEAT_TYPES, + ) -> None: + """ + Feature dimensionality and data type checks + + Arguments: + X (SUPPORTED_FEAT_TYPES): + A set of features that are going to be validated (type and dimensionality + checks) and a encoder fitted in the case the data needs encoding + """ + raise NotImplementedError() + def transform( self, X: SupportedFeatTypes, diff --git a/autoPyTorch/data/tabular_feature_validator.py b/autoPyTorch/data/tabular_feature_validator.py index fab2471c4..f8742c72a 100644 --- a/autoPyTorch/data/tabular_feature_validator.py +++ b/autoPyTorch/data/tabular_feature_validator.py @@ -162,9 +162,13 @@ def _fit( # with nan values. # Columns that are completely made of NaN values are provided to the pipeline # so that later stages decide how to handle them + + # Clear whatever null column markers we had previously + self.null_columns.clear() if np.any(pd.isnull(X)): for column in X.columns: if X[column].isna().all(): + self.null_columns.add(column) X[column] = pd.to_numeric(X[column]) # Also note this change in self.dtypes if len(self.dtypes) != 0: @@ -244,11 +248,25 @@ def transform( if isinstance(X, np.ndarray): X = self.numpy_array_to_pandas(X) - if ispandas(X) and not issparse(X): - if np.any(pd.isnull(X)): - for column in X.columns: - if X[column].isna().all(): - X[column] = pd.to_numeric(X[column]) + if hasattr(X, "iloc") and not issparse(X): + X = cast(pd.DataFrame, X) + # If we had null columns in our fit call and we made them numeric, then: + # - If the columns are null even in transform, apply the same procedure. + # - Otherwise, substitute the values with np.NaN and then make the columns numeric. + # If the column is null here, but it was not in fit, it does not matter. + for column in self.null_columns: + # The column is not null, make it null since it was null in fit. + if not X[column].isna().all(): + X[column] = np.NaN + X[column] = pd.to_numeric(X[column]) + + # for the test set, if we have columns with only null values + # they will probably have a numeric type. If these columns were not + # with only null values in the train set, they should be converted + # to the type that they had during fitting. + for column in X.columns: + if X[column].isna().all(): + X[column] = X[column].astype(self.dtypes[list(X.columns).index(column)]) # Also remove the object dtype for new data if not X.select_dtypes(include='object').empty: @@ -256,18 +274,12 @@ def transform( # Check the data here so we catch problems on new test data self._check_data(X) + # We also need to fillna on the transformation + # in case test data is provided + X = self.impute_nan_in_categories(X) - # Pandas related transformations - if ispandas(X) and self.column_transformer is not None: - if np.any(pd.isnull(X)): - # After above check it means that if there is a NaN - # the whole column must be NaN - # Make sure it is numerical and let the pipeline handle it - for column in X.columns: - if X[column].isna().all(): - X[column] = pd.to_numeric(X[column]) - - X = self.column_transformer.transform(X) + if self.encoder is not None: + X = self.encoder.transform(X) # Sparse related transformations # Not all sparse format support index sorting @@ -557,7 +569,7 @@ def numpy_array_to_pandas( Returns: pd.DataFrame """ - return pd.DataFrame(X).infer_objects().convert_dtypes() + return pd.DataFrame(X).convert_dtypes() def infer_objects(self, X: pd.DataFrame) -> pd.DataFrame: """ @@ -575,18 +587,13 @@ def infer_objects(self, X: pd.DataFrame) -> pd.DataFrame: if hasattr(self, 'object_dtype_mapping'): # Mypy does not process the has attr. This dict is defined below for key, dtype in self.object_dtype_mapping.items(): # type: ignore[has-type] - if 'int' in dtype.name: - # In the case train data was interpreted as int - # and test data was interpreted as float, because of 0.0 - # for example, honor training data - X[key] = X[key].applymap(np.int64) - else: - try: - X[key] = X[key].astype(dtype.name) - except Exception as e: - # Try inference if possible - self.logger.warning(f"Tried to cast column {key} to {dtype} caused {e}") - pass + # honor the training data types + try: + X[key] = X[key].astype(dtype.name) + except Exception as e: + # Try inference if possible + self.logger.warning(f"Tried to cast column {key} to {dtype} caused {e}") + pass else: X = X.infer_objects() for column in X.columns: diff --git a/test/test_data/test_feature_validator.py b/test/test_data/test_feature_validator.py index 08da7d7fd..d4a70c01c 100644 --- a/test/test_data/test_feature_validator.py +++ b/test/test_data/test_feature_validator.py @@ -1,4 +1,4 @@ -import copy + import copy import functools import numpy as np @@ -139,9 +139,9 @@ def test_featurevalidator_fitontypeA_transformtypeB(input_data_featuretest): if isinstance(input_data_featuretest, pd.DataFrame): pytest.skip("Column order change in pandas is not supported") elif isinstance(input_data_featuretest, np.ndarray): - complementary_type = pd.DataFrame(input_data_featuretest) + complementary_type = validator.numpy_array_to_pandas(input_data_featuretest) elif isinstance(input_data_featuretest, list): - complementary_type = pd.DataFrame(input_data_featuretest) + complementary_type, _ = validator.list_to_dataframe(input_data_featuretest) elif sparse.issparse(input_data_featuretest): complementary_type = sparse.csr_matrix(input_data_featuretest.todense()) else: @@ -331,8 +331,11 @@ def test_unknown_encode_value(): ) @pytest.mark.parametrize('train_data_type', ('numpy', 'pandas', 'list')) @pytest.mark.parametrize('test_data_type', ('numpy', 'pandas', 'list')) -def test_featurevalidator_new_data_after_fit(openml_id, - train_data_type, test_data_type): +def test_feature_validator_new_data_after_fit( + openml_id, + train_data_type, + test_data_type, +): # List is currently not supported as infer_objects # cast list objects to type objects @@ -526,3 +529,65 @@ def test_feature_validator_get_columns_to_encode_error_feat_type(input_data_feat validator = TabularFeatureValidator(feat_types=feat_types) with pytest.raises(ValueError, match=r"Expected type of features to be in .*"): validator._validate_feat_types(X) + +def test_feature_validator_imbalanced_data(): + + # Null columns in the train split but not necessarily in the test split + train_features = { + 'A': [np.NaN, np.NaN, np.NaN], + 'B': [1, 2, 3], + 'C': [np.NaN, np.NaN, np.NaN], + 'D': [np.NaN, np.NaN, np.NaN], + } + test_features = { + 'A': [3, 4, 5], + 'B': [6, 5, 7], + 'C': [np.NaN, np.NaN, np.NaN], + 'D': ['Blue', np.NaN, np.NaN], + } + + X_train = pd.DataFrame.from_dict(train_features) + X_test = pd.DataFrame.from_dict(test_features) + validator = TabularFeatureValidator() + validator.fit(X_train) + + train_feature_types = copy.deepcopy(validator.feat_type) + assert train_feature_types == ['numerical', 'numerical', 'numerical', 'numerical'] + # validator will throw an error if the column types are not the same + transformed_X_test = validator.transform(X_test) + transformed_X_test = pd.DataFrame(transformed_X_test) + null_columns = [] + for column in transformed_X_test.columns: + if transformed_X_test[column].isna().all(): + null_columns.append(column) + assert null_columns == [0, 2, 3] + + # Columns with not all null values in the train split and + # completely null on the test split. + train_features = { + 'A': [np.NaN, np.NaN, 4], + 'B': [1, 2, 3], + 'C': ['Blue', np.NaN, np.NaN], + } + test_features = { + 'A': [np.NaN, np.NaN, np.NaN], + 'B': [6, 5, 7], + 'C': [np.NaN, np.NaN, np.NaN], + } + + X_train = pd.DataFrame.from_dict(train_features) + X_test = pd.DataFrame.from_dict(test_features) + validator = TabularFeatureValidator() + validator.fit(X_train) + train_feature_types = copy.deepcopy(validator.feat_type) + assert train_feature_types == ['categorical', 'numerical', 'numerical'] + + transformed_X_test = validator.transform(X_test) + transformed_X_test = pd.DataFrame(transformed_X_test) + null_columns = [] + for column in transformed_X_test.columns: + if transformed_X_test[column].isna().all(): + null_columns.append(column) + + assert null_columns == [1] +>>>>>>> Fixing issues with imbalanced datasets (#197) diff --git a/test/test_data/test_validation.py b/test/test_data/test_validation.py index ba60a1760..341900413 100644 --- a/test/test_data/test_validation.py +++ b/test/test_data/test_validation.py @@ -32,7 +32,6 @@ def test_data_validation_for_classification(openmlid, as_frame): x, y, test_size=0.33, random_state=0) validator.fit(X_train=X_train, y_train=y_train, X_test=X_test, y_test=y_test) - X_train_t, y_train_t = validator.transform(X_train, y_train) assert np.shape(X_train) == np.shape(X_train_t) From 23d808b5a19be1db5cec3525a1820102fe2255b6 Mon Sep 17 00:00:00 2001 From: Ravin Kohli <13005107+ravinkohli@users.noreply.github.com> Date: Tue, 11 May 2021 15:25:27 +0200 Subject: [PATCH 25/50] Reproducibility in cocktail (#204) * Fix randomness in cocktail ingredients * Fix flake --- autoPyTorch/pipeline/components/setup/base_setup.py | 8 +++++--- .../components/setup/network_embedding/NoEmbedding.py | 2 +- .../setup/network_embedding/base_network_embedding.py | 5 ++--- .../components/training/trainer/GridCutMixTrainer.py | 8 ++++---- .../components/training/trainer/GridCutOutTrainer.py | 6 +++--- .../components/training/trainer/RowCutMixTrainer.py | 7 +++---- .../components/training/trainer/RowCutOutTrainer.py | 5 ++--- 7 files changed, 20 insertions(+), 21 deletions(-) diff --git a/autoPyTorch/pipeline/components/setup/base_setup.py b/autoPyTorch/pipeline/components/setup/base_setup.py index 43bb41b56..eff6b6e69 100644 --- a/autoPyTorch/pipeline/components/setup/base_setup.py +++ b/autoPyTorch/pipeline/components/setup/base_setup.py @@ -1,4 +1,6 @@ -from typing import Any, Dict +from typing import Any, Dict, Optional + +import numpy as np from autoPyTorch.pipeline.components.base_component import autoPyTorchComponent @@ -7,8 +9,8 @@ class autoPyTorchSetupComponent(autoPyTorchComponent): """Provide an abstract interface for schedulers in Auto-Pytorch""" - def __init__(self) -> None: - super(autoPyTorchSetupComponent, self).__init__() + def __init__(self, random_state: Optional[np.random.RandomState] = None) -> None: + super(autoPyTorchSetupComponent, self).__init__(random_state=random_state) def transform(self, X: Dict[str, Any]) -> Dict[str, Any]: """ diff --git a/autoPyTorch/pipeline/components/setup/network_embedding/NoEmbedding.py b/autoPyTorch/pipeline/components/setup/network_embedding/NoEmbedding.py index 52c56bc00..8fa03a65e 100644 --- a/autoPyTorch/pipeline/components/setup/network_embedding/NoEmbedding.py +++ b/autoPyTorch/pipeline/components/setup/network_embedding/NoEmbedding.py @@ -24,7 +24,7 @@ class NoEmbedding(NetworkEmbeddingComponent): Class to learn an embedding for categorical hyperparameters. """ - def __init__(self, random_state: Optional[Union[np.random.RandomState, int]] = None): + def __init__(self, random_state: Optional[np.random.RandomState] = None): super().__init__(random_state=random_state) def build_embedding(self, diff --git a/autoPyTorch/pipeline/components/setup/network_embedding/base_network_embedding.py b/autoPyTorch/pipeline/components/setup/network_embedding/base_network_embedding.py index 1ff5df13e..e113d5774 100644 --- a/autoPyTorch/pipeline/components/setup/network_embedding/base_network_embedding.py +++ b/autoPyTorch/pipeline/components/setup/network_embedding/base_network_embedding.py @@ -11,10 +11,9 @@ class NetworkEmbeddingComponent(autoPyTorchSetupComponent): - def __init__(self, random_state: Optional[Union[np.random.RandomState, int]] = None): - super().__init__() + def __init__(self, random_state: Optional[np.random.RandomState] = None): + super().__init__(random_state=random_state) self.embedding: Optional[nn.Module] = None - self.random_state = random_state self.feature_shapes: Dict[str, int] = {} def fit(self, X: Dict[str, Any], y: Any = None) -> BaseEstimator: diff --git a/autoPyTorch/pipeline/components/training/trainer/GridCutMixTrainer.py b/autoPyTorch/pipeline/components/training/trainer/GridCutMixTrainer.py index fb22e7cb8..71a170c61 100644 --- a/autoPyTorch/pipeline/components/training/trainer/GridCutMixTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/GridCutMixTrainer.py @@ -27,11 +27,11 @@ def data_preparation(self, X: np.ndarray, y: np.ndarray, typing.Dict[str, np.ndarray]: arguments to the criterion function """ beta = 1.0 - lam = np.random.beta(beta, beta) + lam = self.random_state.beta(beta, beta) batch_size, channel, W, H = X.size() index = torch.randperm(batch_size).cuda() if X.is_cuda else torch.randperm(batch_size) - r = np.random.rand(1) + r = self.random_state.rand(1) if beta <= 0 or r > self.alpha: return X, {'y_a': y, 'y_b': y[index], 'lam': 1} @@ -40,8 +40,8 @@ def data_preparation(self, X: np.ndarray, y: np.ndarray, cut_rat = np.sqrt(1. - lam) cut_w = np.int(W * cut_rat) cut_h = np.int(H * cut_rat) - cx = np.random.randint(W) - cy = np.random.randint(H) + cx = self.random_state.randint(W) + cy = self.random_state.randint(H) bbx1 = np.clip(cx - cut_w // 2, 0, W) bby1 = np.clip(cy - cut_h // 2, 0, H) bbx2 = np.clip(cx + cut_w // 2, 0, W) diff --git a/autoPyTorch/pipeline/components/training/trainer/GridCutOutTrainer.py b/autoPyTorch/pipeline/components/training/trainer/GridCutOutTrainer.py index 37c71d53b..b2fd6151a 100644 --- a/autoPyTorch/pipeline/components/training/trainer/GridCutOutTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/GridCutOutTrainer.py @@ -24,7 +24,7 @@ def data_preparation(self, X: np.ndarray, y: np.ndarray, np.ndarray: that processes data typing.Dict[str, np.ndarray]: arguments to the criterion function """ - r = np.random.rand(1) + r = self.random_state.rand(1) batch_size, channel, W, H = X.size() if r > self.cutout_prob: return X, {'y_a': y, 'y_b': y, 'lam': 1} @@ -34,8 +34,8 @@ def data_preparation(self, X: np.ndarray, y: np.ndarray, cut_rat = np.sqrt(1. - self.patch_ratio) cut_w = np.int(W * cut_rat) cut_h = np.int(H * cut_rat) - cx = np.random.randint(W) - cy = np.random.randint(H) + cx = self.random_state.randint(W) + cy = self.random_state.randint(H) bbx1 = np.clip(cx - cut_w // 2, 0, W) bby1 = np.clip(cy - cut_h // 2, 0, H) bbx2 = np.clip(cx + cut_w // 2, 0, W) diff --git a/autoPyTorch/pipeline/components/training/trainer/RowCutMixTrainer.py b/autoPyTorch/pipeline/components/training/trainer/RowCutMixTrainer.py index b639156bb..f0d8536f9 100644 --- a/autoPyTorch/pipeline/components/training/trainer/RowCutMixTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/RowCutMixTrainer.py @@ -1,4 +1,3 @@ -import random import typing import numpy as np @@ -28,11 +27,11 @@ def data_preparation(self, X: np.ndarray, y: np.ndarray, typing.Dict[str, np.ndarray]: arguments to the criterion function """ beta = 1.0 - lam = np.random.beta(beta, beta) + lam = self.random_state.beta(beta, beta) batch_size = X.size()[0] index = torch.randperm(batch_size).cuda() if X.is_cuda else torch.randperm(batch_size) - r = np.random.rand(1) + r = self.random_state.rand(1) if beta <= 0 or r > self.alpha: return X, {'y_a': y, 'y_b': y[index], 'lam': 1} @@ -40,7 +39,7 @@ def data_preparation(self, X: np.ndarray, y: np.ndarray, # It is unlikely that the batch size is lower than the number of features, but # be safe size = min(X.shape[0], X.shape[1]) - indices = torch.tensor(random.sample(range(1, size), max(1, np.int(size * lam)))) + indices = torch.tensor(self.random_state.choice(range(1, size), max(1, np.int(size * lam)))) X[:, indices] = X[index, :][:, indices] diff --git a/autoPyTorch/pipeline/components/training/trainer/RowCutOutTrainer.py b/autoPyTorch/pipeline/components/training/trainer/RowCutOutTrainer.py index 660f6202f..a7936c4f8 100644 --- a/autoPyTorch/pipeline/components/training/trainer/RowCutOutTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/RowCutOutTrainer.py @@ -1,4 +1,3 @@ -import random import typing import numpy as np @@ -28,7 +27,7 @@ def data_preparation(self, X: np.ndarray, y: np.ndarray, typing.Dict[str, np.ndarray]: arguments to the criterion function """ - r = np.random.rand(1) + r = self.random_state.rand(1) if r > self.cutout_prob: y_a = y y_b = y @@ -39,7 +38,7 @@ def data_preparation(self, X: np.ndarray, y: np.ndarray, # It is unlikely that the batch size is lower than the number of features, but # be safe size = min(X.shape[0], X.shape[1]) - indices = torch.tensor(random.sample(range(1, size), max(1, np.int(size * self.patch_ratio)))) + indices = torch.tensor(self.random_state.choice(range(1, size), max(1, np.int(size * self.patch_ratio)))) # We use an ordinal encoder on the tabular data # -1 is the conceptual equivalent to 0 in a image, that does not From 6283c56b6c0d7e050f36a74ffc130247a95da1fb Mon Sep 17 00:00:00 2001 From: Ravin Kohli <13005107+ravinkohli@users.noreply.github.com> Date: Tue, 11 May 2021 17:38:45 +0200 Subject: [PATCH 26/50] fix bug in adversarial trainer (#207) --- .../training/trainer/AdversarialTrainer.py | 37 +++++++++++++------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py b/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py index d96e2117d..4d5163064 100644 --- a/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py @@ -196,22 +196,35 @@ def get_hyperparameter_search_space( add_hyperparameter(cs, epsilon, UniformFloatHyperparameter) add_hyperparameter(cs, use_stochastic_weight_averaging, CategoricalHyperparameter) + snapshot_ensemble_flag = False + if any(use_snapshot_ensemble.value_range): + snapshot_ensemble_flag = True + use_snapshot_ensemble = get_hyperparameter(use_snapshot_ensemble, CategoricalHyperparameter) - se_lastk = get_hyperparameter(se_lastk, Constant) - cs.add_hyperparameters([use_snapshot_ensemble, se_lastk]) - cond = EqualsCondition(se_lastk, use_snapshot_ensemble, True) - cs.add_condition(cond) + cs.add_hyperparameter(use_snapshot_ensemble) + + if snapshot_ensemble_flag: + se_lastk = get_hyperparameter(se_lastk, Constant) + cs.add_hyperparameter(se_lastk) + cond = EqualsCondition(se_lastk, use_snapshot_ensemble, True) + cs.add_condition(cond) + + lookahead_flag = False + if any(use_lookahead_optimizer.value_range): + lookahead_flag = True use_lookahead_optimizer = get_hyperparameter(use_lookahead_optimizer, CategoricalHyperparameter) cs.add_hyperparameter(use_lookahead_optimizer) - la_config_space = Lookahead.get_hyperparameter_search_space(la_steps=la_steps, - la_alpha=la_alpha) - parent_hyperparameter = {'parent': use_lookahead_optimizer, 'value': True} - cs.add_configuration_space( - Lookahead.__name__, - la_config_space, - parent_hyperparameter=parent_hyperparameter - ) + + if lookahead_flag: + la_config_space = Lookahead.get_hyperparameter_search_space(la_steps=la_steps, + la_alpha=la_alpha) + parent_hyperparameter = {'parent': use_lookahead_optimizer, 'value': True} + cs.add_configuration_space( + Lookahead.__name__, + la_config_space, + parent_hyperparameter=parent_hyperparameter + ) """ if dataset_properties is not None: From bc0540b82d6bb41d810b1bf8571afeb237d81f1e Mon Sep 17 00:00:00 2001 From: Ravin Kohli <13005107+ravinkohli@users.noreply.github.com> Date: Fri, 14 May 2021 14:13:37 +0200 Subject: [PATCH 27/50] Add dropout shape as a hyperparameter (#213) * Add dropout shape as a hyperparameter * fix stupid bug --- .../network_backbone/ShapedResNetBackbone.py | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py index 6383e6230..1143177b8 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py @@ -45,12 +45,12 @@ def build_backbone(self, input_shape: Tuple[int, ...]) -> torch.nn.Sequential: # n_units for the architecture, since, it is mostly implemented for the # output layer, which is part of the head and not of the backbone. dropout_shape = get_shaped_neuron_counts( - shape=self.config['resnet_shape'], - in_feat=0, - out_feat=0, - max_neurons=self.config["max_dropout"], - layer_count=self.config['num_groups'] + 1, - )[:-1] + self.config['dropout_shape'], 0, 0, 1000, self.config['num_groups'] + ) + + dropout_shape = [ + dropout / 1000 * self.config["max_dropout"] for dropout in dropout_shape + ] self.config.update( {"dropout_%d" % (i + 1): dropout for i, dropout in enumerate(dropout_shape)} @@ -136,6 +136,13 @@ def get_hyperparameter_search_space( # type: ignore[override] max_dropout: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="max_dropout", value_range=(0, 0.8), default_value=0.5), + dropout_shape: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="dropout_shape", + value_range=('funnel', 'long_funnel', + 'diamond', 'hexagon', + 'brick', 'triangle', + 'stairs'), + default_value='funnel', + ), max_shake_drop_probability: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="max_shake_drop_probability", value_range=(0, 1), @@ -165,8 +172,10 @@ def get_hyperparameter_search_space( # type: ignore[override] if dropout_flag: max_dropout = get_hyperparameter(max_dropout, UniformFloatHyperparameter) - cs.add_hyperparameter(max_dropout) + dropout_shape = get_hyperparameter(dropout_shape, CategoricalHyperparameter) + cs.add_hyperparameters([dropout_shape, max_dropout]) cs.add_condition(CS.EqualsCondition(max_dropout, use_dropout, True)) + cs.add_condition(CS.EqualsCondition(dropout_shape, use_dropout, True)) skip_connection_flag = False if any(use_skip_connection.value_range): From 5d6062f69049a53d4e3c1902291cd47e8c9b00f2 Mon Sep 17 00:00:00 2001 From: Ravin Kohli <13005107+ravinkohli@users.noreply.github.com> Date: Fri, 14 May 2021 17:09:50 +0200 Subject: [PATCH 28/50] Change weighted loss to categorical and fix for test adversarial trainer (#214) --- .../training/trainer/AdversarialTrainer.py | 15 ++++----------- .../training/trainer/StandardTrainer.py | 3 ++- .../components/training/trainer/base_trainer.py | 17 +++++------------ .../components/training/trainer/cutout_utils.py | 16 ++++------------ .../components/training/trainer/mixup_utils.py | 16 ++++------------ .../test_tabular_classification.py | 2 +- 6 files changed, 20 insertions(+), 49 deletions(-) diff --git a/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py b/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py index 4d5163064..af000115f 100644 --- a/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py @@ -24,7 +24,7 @@ class AdversarialTrainer(BaseTrainerComponent): def __init__( self, epsilon: float, - weighted_loss: int = 0, + weighted_loss: bool = False, random_state: Optional[np.random.RandomState] = None, use_stochastic_weight_averaging: bool = False, use_snapshot_ensemble: bool = False, @@ -159,8 +159,8 @@ def get_hyperparameter_search_space( dataset_properties: Optional[Dict] = None, weighted_loss: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="weighted_loss", - value_range=[1], - default_value=1), + value_range=[True, False], + default_value=True), la_steps: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="la_steps", value_range=(5, 10), @@ -226,16 +226,9 @@ def get_hyperparameter_search_space( parent_hyperparameter=parent_hyperparameter ) - """ + # TODO, decouple the weighted loss from the trainer if dataset_properties is not None: if STRING_TO_TASK_TYPES[dataset_properties['task_type']] in CLASSIFICATION_TASKS: add_hyperparameter(cs, weighted_loss, CategoricalHyperparameter) - """ - # TODO, decouple the weighted loss from the trainer. Uncomment the code above and - # remove the code below. Also update the method signature, so the weighted loss - # is not a constant. - if dataset_properties is not None: - if STRING_TO_TASK_TYPES[dataset_properties['task_type']] in CLASSIFICATION_TASKS: - add_hyperparameter(cs, weighted_loss, Constant) return cs diff --git a/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py b/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py index 4d6cf52a1..9e44399fd 100644 --- a/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py @@ -13,7 +13,8 @@ class StandardTrainer(BaseTrainerComponent): - def __init__(self, weighted_loss: int = 0, + def __init__(self, + weighted_loss: bool = False, use_stochastic_weight_averaging: bool = False, use_snapshot_ensemble: bool = False, se_lastk: int = 3, diff --git a/autoPyTorch/pipeline/components/training/trainer/base_trainer.py b/autoPyTorch/pipeline/components/training/trainer/base_trainer.py index 22535de55..96fdd4fea 100644 --- a/autoPyTorch/pipeline/components/training/trainer/base_trainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/base_trainer.py @@ -213,7 +213,7 @@ class BaseTrainerComponent(autoPyTorchTrainingComponent): """ Base class for training Args: - weighted_loss (int, default=0): In case for classification, whether to weight + weighted_loss (bool, default=False): In case for classification, whether to weight the loss function according to the distribution of classes in the target use_stochastic_weight_averaging (bool, default=True): whether to use stochastic weight averaging. Stochastic weight averaging is a simple average of @@ -228,7 +228,7 @@ class BaseTrainerComponent(autoPyTorchTrainingComponent): random_state: **lookahead_config: """ - def __init__(self, weighted_loss: int = 0, + def __init__(self, weighted_loss: bool = False, use_stochastic_weight_averaging: bool = True, use_snapshot_ensemble: bool = True, se_lastk: int = 3, @@ -596,8 +596,8 @@ def get_hyperparameter_search_space( dataset_properties: Optional[Dict] = None, weighted_loss: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="weighted_loss", - value_range=[1], - default_value=1), + value_range=[True, False], + default_value=True), la_steps: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="la_steps", value_range=(5, 10), @@ -645,16 +645,9 @@ def get_hyperparameter_search_space( parent_hyperparameter=parent_hyperparameter ) - """ + # TODO, decouple the weighted loss from the trainer if dataset_properties is not None: if STRING_TO_TASK_TYPES[dataset_properties['task_type']] in CLASSIFICATION_TASKS: add_hyperparameter(cs, weighted_loss, CategoricalHyperparameter) - """ - # TODO, decouple the weighted loss from the trainer. Uncomment the code above and - # remove the code below. Also update the method signature, so the weighted loss - # is not a constant. - if dataset_properties is not None: - if STRING_TO_TASK_TYPES[dataset_properties['task_type']] in CLASSIFICATION_TASKS: - add_hyperparameter(cs, weighted_loss, Constant) return cs diff --git a/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py b/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py index d073ea6cf..582014f9b 100644 --- a/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py +++ b/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py @@ -20,7 +20,7 @@ class CutOut: def __init__(self, patch_ratio: float, cutout_prob: float, - weighted_loss: int = 0, + weighted_loss: bool = False, random_state: Optional[np.random.RandomState] = None, use_stochastic_weight_averaging: bool = False, use_snapshot_ensemble: bool = False, @@ -63,9 +63,8 @@ def get_hyperparameter_search_space( dataset_properties: Optional[Dict] = None, weighted_loss: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="weighted_loss", - value_range=[1], - default_value=1 - ), + value_range=[True, False], + default_value=True), la_steps: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="la_steps", value_range=(5, 10), @@ -137,16 +136,9 @@ def get_hyperparameter_search_space( parent_hyperparameter=parent_hyperparameter ) - """ + # TODO, decouple the weighted loss from the trainer if dataset_properties is not None: if STRING_TO_TASK_TYPES[dataset_properties['task_type']] in CLASSIFICATION_TASKS: add_hyperparameter(cs, weighted_loss, CategoricalHyperparameter) - """ - # TODO, decouple the weighted loss from the trainer. Uncomment the code above and - # remove the code below. Also update the method signature, so the weighted loss - # is not a constant. - if dataset_properties is not None: - if STRING_TO_TASK_TYPES[dataset_properties['task_type']] in CLASSIFICATION_TASKS: - add_hyperparameter(cs, weighted_loss, Constant) return cs diff --git a/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py b/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py index ada279658..a0348a566 100644 --- a/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py +++ b/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py @@ -19,7 +19,7 @@ class MixUp: def __init__(self, alpha: float, - weighted_loss: int = 0, + weighted_loss: bool = False, random_state: Optional[np.random.RandomState] = None, use_stochastic_weight_averaging: bool = False, use_snapshot_ensemble: bool = False, @@ -61,9 +61,8 @@ def get_hyperparameter_search_space( dataset_properties: Optional[Dict] = None, weighted_loss: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="weighted_loss", - value_range=[1], - default_value=1 - ), + value_range=[True, False], + default_value=True), la_steps: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="la_steps", value_range=(5, 10), @@ -128,16 +127,9 @@ def get_hyperparameter_search_space( la_config_space, parent_hyperparameter=parent_hyperparameter ) - """ + # TODO, decouple the weighted loss from the trainer if dataset_properties is not None: if STRING_TO_TASK_TYPES[dataset_properties['task_type']] in CLASSIFICATION_TASKS: add_hyperparameter(cs, weighted_loss, CategoricalHyperparameter) - """ - # TODO, decouple the weighted loss from the trainer. Uncomment the code above and - # remove the code below. Also update the method signature, so the weighted loss - # is not a constant. - if dataset_properties is not None: - if STRING_TO_TASK_TYPES[dataset_properties['task_type']] in CLASSIFICATION_TASKS: - add_hyperparameter(cs, weighted_loss, Constant) return cs diff --git a/test/test_pipeline/test_tabular_classification.py b/test/test_pipeline/test_tabular_classification.py index cc89be942..b0923fd41 100644 --- a/test/test_pipeline/test_tabular_classification.py +++ b/test/test_pipeline/test_tabular_classification.py @@ -398,7 +398,7 @@ def test_set_choices_updates(self, fit_dictionary_tabular): @pytest.mark.parametrize('lr_scheduler', ['CosineAnnealingWarmRestarts', 'ReduceLROnPlateau']) def test_trainer_cocktails(self, fit_dictionary_tabular, mocker, lr_scheduler, trainer): # noqa F811 - fit_dictionary_tabular['epochs'] = 20 + fit_dictionary_tabular['epochs'] = 45 fit_dictionary_tabular['early_stopping'] = 20 pipeline = TabularClassificationPipeline( dataset_properties=fit_dictionary_tabular['dataset_properties'], From 622c185fd69e99876661984e37af37b97924a10a Mon Sep 17 00:00:00 2001 From: Ravin Kohli <13005107+ravinkohli@users.noreply.github.com> Date: Mon, 17 May 2021 13:09:17 +0200 Subject: [PATCH 29/50] added no head (#218) --- .../components/setup/network_head/no_head.py | 52 +++++++++++++++++++ .../components/setup/test_setup_networks.py | 2 +- 2 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 autoPyTorch/pipeline/components/setup/network_head/no_head.py diff --git a/autoPyTorch/pipeline/components/setup/network_head/no_head.py b/autoPyTorch/pipeline/components/setup/network_head/no_head.py new file mode 100644 index 000000000..f5cadb416 --- /dev/null +++ b/autoPyTorch/pipeline/components/setup/network_head/no_head.py @@ -0,0 +1,52 @@ +from typing import Any, Dict, Optional, Tuple, Union + +from ConfigSpace.configuration_space import ConfigurationSpace +from ConfigSpace.hyperparameters import CategoricalHyperparameter + +import numpy as np + +from torch import nn + +from autoPyTorch.pipeline.components.setup.network_head.base_network_head import NetworkHeadComponent +from autoPyTorch.pipeline.components.setup.network_head.utils import _activations +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter + + +class NoHead(NetworkHeadComponent): + """ + Head which only adds a fully connected layer which takes the + output of the backbone as input and outputs the predictions. + Flattens any input in a array of shape [B, prod(input_shape)]. + """ + + def build_head(self, input_shape: Tuple[int, ...], output_shape: Tuple[int, ...]) -> nn.Module: + layers = [nn.Flatten()] + in_features = np.prod(input_shape).item() + out_features = np.prod(output_shape).item() + layers.append(_activations[self.config["activation"]]()) + layers.append(nn.Linear(in_features=in_features, + out_features=out_features)) + return nn.Sequential(*layers) + + @staticmethod + def get_properties(dataset_properties: Optional[Dict[str, Any]] = None) -> Dict[str, Union[str, bool]]: + return { + 'shortname': 'NoHead', + 'name': 'NoHead', + 'handles_tabular': True, + 'handles_image': True, + 'handles_time_series': True, + } + + @staticmethod + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict[str, str]] = None, + activation: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="activation", + value_range=tuple(_activations.keys()), + default_value=list(_activations.keys())[0]), + ) -> ConfigurationSpace: + cs = ConfigurationSpace() + + add_hyperparameter(cs, activation, CategoricalHyperparameter) + + return cs diff --git a/test/test_pipeline/components/setup/test_setup_networks.py b/test/test_pipeline/components/setup/test_setup_networks.py index e8f3f7da8..f3b9ff11c 100644 --- a/test/test_pipeline/components/setup/test_setup_networks.py +++ b/test/test_pipeline/components/setup/test_setup_networks.py @@ -14,7 +14,7 @@ def backbone(request): return request.param -@pytest.fixture(params=['fully_connected']) +@pytest.fixture(params=['fully_connected', 'no_head']) def head(request): return request.param From c4b7729376a79576e0f1ba2ee630e212cb09ae59 Mon Sep 17 00:00:00 2001 From: Ravin Kohli <13005107+ravinkohli@users.noreply.github.com> Date: Fri, 21 May 2021 21:46:09 +0200 Subject: [PATCH 30/50] Fix bugs in cutout training (#233) * Fix bugs in cutout training * Address comments from arlind --- .../training/trainer/RowCutMixTrainer.py | 8 +++--- .../training/trainer/RowCutOutTrainer.py | 27 +++++++++++++------ .../components/training/trainer/__init__.py | 13 ++------- .../training/trainer/base_trainer.py | 4 +++ 4 files changed, 28 insertions(+), 24 deletions(-) diff --git a/autoPyTorch/pipeline/components/training/trainer/RowCutMixTrainer.py b/autoPyTorch/pipeline/components/training/trainer/RowCutMixTrainer.py index f0d8536f9..20d02c793 100644 --- a/autoPyTorch/pipeline/components/training/trainer/RowCutMixTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/RowCutMixTrainer.py @@ -35,11 +35,9 @@ def data_preparation(self, X: np.ndarray, y: np.ndarray, if beta <= 0 or r > self.alpha: return X, {'y_a': y, 'y_b': y[index], 'lam': 1} - # The mixup component mixes up also on the batch dimension - # It is unlikely that the batch size is lower than the number of features, but - # be safe - size = min(X.shape[0], X.shape[1]) - indices = torch.tensor(self.random_state.choice(range(1, size), max(1, np.int(size * lam)))) + size = X.shape[1] + indices = torch.tensor(self.random_state.choice(range(1, size), max(1, np.int32(size * lam)), + replace=False)) X[:, indices] = X[index, :][:, indices] diff --git a/autoPyTorch/pipeline/components/training/trainer/RowCutOutTrainer.py b/autoPyTorch/pipeline/components/training/trainer/RowCutOutTrainer.py index a7936c4f8..97f0caa18 100644 --- a/autoPyTorch/pipeline/components/training/trainer/RowCutOutTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/RowCutOutTrainer.py @@ -9,6 +9,8 @@ class RowCutOutTrainer(CutOut, BaseTrainerComponent): + NUMERICAL_VALUE = 0 + CATEGORICAL_VALUE = -1 def data_preparation(self, X: np.ndarray, y: np.ndarray, ) -> typing.Tuple[np.ndarray, typing.Dict[str, np.ndarray]]: @@ -34,17 +36,26 @@ def data_preparation(self, X: np.ndarray, y: np.ndarray, lam = 1 return X, {'y_a': y_a, 'y_b': y_b, 'lam': lam} - # The mixup component mixes up also on the batch dimension - # It is unlikely that the batch size is lower than the number of features, but - # be safe - size = min(X.shape[0], X.shape[1]) - indices = torch.tensor(self.random_state.choice(range(1, size), max(1, np.int(size * self.patch_ratio)))) + size = X.shape[1] + indices = self.random_state.choice(range(1, size), max(1, np.int32(size * self.patch_ratio)), + replace=False) - # We use an ordinal encoder on the tabular data + if not isinstance(self.numerical_columns, typing.Iterable): + raise ValueError("{} requires numerical columns information of {}" + "to prepare data got {}.".format(self.__class__.__name__, + typing.Iterable, + self.numerical_columns)) + numerical_indices = torch.tensor(self.numerical_columns) + categorical_indices = torch.tensor([index for index in indices if index not in self.numerical_columns]) + + # We use an ordinal encoder on the categorical columns of tabular data # -1 is the conceptual equivalent to 0 in a image, that does not # have color as a feature and hence the network has to learn to deal - # without this data - X[:, indices.long()] = -1 + # without this data. For numerical columns we use 0 to cutout the features + # similar to the effect that setting 0 as a pixel value in an image. + X[:, categorical_indices.long()] = self.CATEGORICAL_VALUE + X[:, numerical_indices.long()] = self.NUMERICAL_VALUE + lam = 1 y_a = y y_b = y diff --git a/autoPyTorch/pipeline/components/training/trainer/__init__.py b/autoPyTorch/pipeline/components/training/trainer/__init__.py index b246cc0c2..cd53ebcaa 100755 --- a/autoPyTorch/pipeline/components/training/trainer/__init__.py +++ b/autoPyTorch/pipeline/components/training/trainer/__init__.py @@ -319,6 +319,8 @@ def prepare_trainer(self, X: Dict) -> None: task_type=STRING_TO_TASK_TYPES[X['dataset_properties']['task_type']], labels=labels, step_interval=X['step_interval'] + numerical_columns=X['dataset_properties']['numerical_columns'] if 'numerical_columns' in X[ + 'dataset_properties'] else None ) def get_budget_tracker(self, X: Dict) -> BudgetTracker: @@ -396,11 +398,7 @@ def _fit(self, X: Dict[str, Any], y: Any = None, **kwargs: Any) -> 'TrainerChoic val_loss, val_metrics, test_loss, test_metrics = None, {}, None, {} if self.eval_valid_each_epoch(X): -<<<<<<< HEAD - if X['val_data_loader']: -======= if 'val_data_loader' in X and X['val_data_loader']: ->>>>>>> Create fit evaluator, no resampling strategy and fix bug for test statistics val_loss, val_metrics = self.choice.evaluate(X['val_data_loader'], epoch, writer) if 'test_data_loader' in X and X['test_data_loader']: test_loss, test_metrics = self.choice.evaluate(X['test_data_loader'], epoch, writer) @@ -454,17 +452,10 @@ def _fit(self, X: Dict[str, Any], y: Any = None, **kwargs: Any) -> 'TrainerChoic # wrap up -- add score if not evaluating every epoch if not self.eval_valid_each_epoch(X): -<<<<<<< HEAD - if X['val_data_loader']: - val_loss, val_metrics = self.choice.evaluate(X['val_data_loader'], epoch, writer) - if 'test_data_loader' in X and X['val_data_loader']: - test_loss, test_metrics = self.choice.evaluate(X['test_data_loader'], epoch, writer) -======= if 'val_data_loader' in X and X['val_data_loader']: val_loss, val_metrics = self.choice.evaluate(X['val_data_loader'], epoch, writer) if 'test_data_loader' in X and X['test_data_loader']: test_loss, test_metrics = self.choice.evaluate(X['test_data_loader']) ->>>>>>> Create fit evaluator, no resampling strategy and fix bug for test statistics self.run_summary.add_performance( epoch=epoch, start_time=start_time, diff --git a/autoPyTorch/pipeline/components/training/trainer/base_trainer.py b/autoPyTorch/pipeline/components/training/trainer/base_trainer.py index 96fdd4fea..145deed00 100644 --- a/autoPyTorch/pipeline/components/training/trainer/base_trainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/base_trainer.py @@ -273,6 +273,7 @@ def prepare( task_type: int, labels: Union[np.ndarray, torch.Tensor, pd.DataFrame], step_interval: Union[str, StepIntervalUnit] = StepIntervalUnit.batch, + numerical_columns: Optional[List[int]] = None, **kwargs: Dict ) -> None: @@ -330,6 +331,9 @@ def prepare( # task type (used for calculating metrics) self.task_type = task_type + # for cutout trainer, we need the list of numerical columns + self.numerical_columns = numerical_columns + def on_epoch_start(self, X: Dict[str, Any], epoch: int) -> None: """ Optional place holder for AutoPytorch Extensions. From 0c8d2ff67571fa67ea17c80e2cc29d489dab44b2 Mon Sep 17 00:00:00 2001 From: Arlind Kadra Date: Thu, 3 Jun 2021 16:33:10 +0200 Subject: [PATCH 31/50] Cocktail hotfixes (#245) * Fixes for the development branch and regularization cocktails * Update implementation * Fix unit tests temporarily * Implementation update and bug fixes * Removing unecessary code * Addressing Ravin's comments [refactor] Address Shuhei's comments [refactor] Address Shuhei's comments [refactor] Address Shuhei's comments [refactor] Address Shuhei's comments --- autoPyTorch/api/base_task.py | 2 +- autoPyTorch/api/tabular_classification.py | 8 - autoPyTorch/api/tabular_regression.py | 8 - autoPyTorch/data/tabular_feature_validator.py | 4 +- autoPyTorch/evaluation/fit_evaluator.py | 319 ++++++++++++++++++ autoPyTorch/evaluation/tae.py | 15 + autoPyTorch/evaluation/train_evaluator.py | 12 +- .../setup/network_backbone/ResNetBackbone.py | 18 +- .../network_backbone/ShapedResNetBackbone.py | 24 +- .../components/setup/network_head/no_head.py | 6 +- .../components/setup/test_setup.py | 7 +- 11 files changed, 377 insertions(+), 46 deletions(-) create mode 100644 autoPyTorch/evaluation/fit_evaluator.py diff --git a/autoPyTorch/api/base_task.py b/autoPyTorch/api/base_task.py index 07d27c6d4..83f3840ba 100644 --- a/autoPyTorch/api/base_task.py +++ b/autoPyTorch/api/base_task.py @@ -459,7 +459,7 @@ def set_pipeline_config(self, **pipeline_config_kwargs: Any) -> None: None """ unknown_keys = [] - for option, value in pipeline_config_kwargs.items(): + for option in pipeline_config_kwargs.keys(): if option in self.pipeline_options.keys(): pass else: diff --git a/autoPyTorch/api/tabular_classification.py b/autoPyTorch/api/tabular_classification.py index 766662fcd..facb59f99 100644 --- a/autoPyTorch/api/tabular_classification.py +++ b/autoPyTorch/api/tabular_classification.py @@ -92,17 +92,9 @@ def __init__( output_directory: Optional[str] = None, delete_tmp_folder_after_terminate: bool = True, delete_output_folder_after_terminate: bool = True, -<<<<<<< HEAD include_components: Optional[Dict[str, Any]] = None, exclude_components: Optional[Dict[str, Any]] = None, resampling_strategy: ResamplingStrategies = HoldoutValTypes.holdout_validation, -======= - include_components: Optional[Dict] = None, - exclude_components: Optional[Dict] = None, - resampling_strategy: Union[CrossValTypes, - HoldoutValTypes, - NoResamplingStrategyTypes] = HoldoutValTypes.holdout_validation, ->>>>>>> Create fit evaluator, no resampling strategy and fix bug for test statistics resampling_strategy_args: Optional[Dict[str, Any]] = None, backend: Optional[Backend] = None, search_space_updates: Optional[HyperparameterSearchSpaceUpdates] = None diff --git a/autoPyTorch/api/tabular_regression.py b/autoPyTorch/api/tabular_regression.py index cec05aca2..e0c1e4eac 100644 --- a/autoPyTorch/api/tabular_regression.py +++ b/autoPyTorch/api/tabular_regression.py @@ -93,17 +93,9 @@ def __init__( output_directory: Optional[str] = None, delete_tmp_folder_after_terminate: bool = True, delete_output_folder_after_terminate: bool = True, -<<<<<<< HEAD include_components: Optional[Dict[str, Any]] = None, exclude_components: Optional[Dict[str, Any]] = None, resampling_strategy: ResamplingStrategies = HoldoutValTypes.holdout_validation, -======= - include_components: Optional[Dict] = None, - exclude_components: Optional[Dict] = None, - resampling_strategy:Union[CrossValTypes, - HoldoutValTypes, - NoResamplingStrategyTypes] = HoldoutValTypes.holdout_validation, ->>>>>>> Create fit evaluator, no resampling strategy and fix bug for test statistics resampling_strategy_args: Optional[Dict[str, Any]] = None, backend: Optional[Backend] = None, search_space_updates: Optional[HyperparameterSearchSpaceUpdates] = None diff --git a/autoPyTorch/data/tabular_feature_validator.py b/autoPyTorch/data/tabular_feature_validator.py index f8742c72a..5e6013e90 100644 --- a/autoPyTorch/data/tabular_feature_validator.py +++ b/autoPyTorch/data/tabular_feature_validator.py @@ -468,7 +468,7 @@ def _get_columns_to_encode( feat_types = [] # Make sure each column is a valid type - for i, column in enumerate(X.columns): + for column in X.columns: if X[column].dtype.name in ['category', 'bool']: transformed_columns.append(column) @@ -592,7 +592,7 @@ def infer_objects(self, X: pd.DataFrame) -> pd.DataFrame: X[key] = X[key].astype(dtype.name) except Exception as e: # Try inference if possible - self.logger.warning(f"Tried to cast column {key} to {dtype} caused {e}") + self.logger.warning(f'Casting the column {key} to {dtype} caused the exception {e}') pass else: X = X.infer_objects() diff --git a/autoPyTorch/evaluation/fit_evaluator.py b/autoPyTorch/evaluation/fit_evaluator.py new file mode 100644 index 000000000..281913003 --- /dev/null +++ b/autoPyTorch/evaluation/fit_evaluator.py @@ -0,0 +1,319 @@ +import time +from multiprocessing.queues import Queue +from typing import Any, Dict, List, Optional, Tuple, Union + +from ConfigSpace.configuration_space import Configuration + +import numpy as np + +from sklearn.base import BaseEstimator + +from smac.tae import StatusType + +from autoPyTorch.datasets.resampling_strategy import NoResamplingStrategyTypes +from autoPyTorch.evaluation.abstract_evaluator import ( + AbstractEvaluator, + fit_and_suppress_warnings +) +from autoPyTorch.pipeline.components.training.metrics.base import autoPyTorchMetric +from autoPyTorch.utils.backend import Backend +from autoPyTorch.utils.common import subsampler +from autoPyTorch.utils.hyperparameter_search_space_update import HyperparameterSearchSpaceUpdates + + +class FitEvaluator(AbstractEvaluator): + def __init__(self, backend: Backend, queue: Queue, + metric: autoPyTorchMetric, + budget: float, + budget_type: str = None, + pipeline_config: Optional[Dict[str, Any]] = None, + configuration: Optional[Configuration] = None, + seed: int = 1, + output_y_hat_optimization: bool = False, + num_run: Optional[int] = None, + include: Optional[Dict[str, Any]] = None, + exclude: Optional[Dict[str, Any]] = None, + disable_file_output: Union[bool, List] = False, + init_params: Optional[Dict[str, Any]] = None, + logger_port: Optional[int] = None, + keep_models: Optional[bool] = None, + all_supported_metrics: bool = True, + search_space_updates: Optional[HyperparameterSearchSpaceUpdates] = None) -> None: + super().__init__( + backend=backend, + queue=queue, + configuration=configuration, + metric=metric, + seed=seed, + output_y_hat_optimization=output_y_hat_optimization, + num_run=num_run, + include=include, + exclude=exclude, + disable_file_output=disable_file_output, + init_params=init_params, + budget=budget, + budget_type=budget_type, + logger_port=logger_port, + all_supported_metrics=all_supported_metrics, + pipeline_config=pipeline_config, + search_space_updates=search_space_updates + ) + if not isinstance(self.datamanager.resampling_strategy, NoResamplingStrategyTypes): + raise ValueError( + "FitEvaluator needs to be fitted on the whole dataset and resampling_strategy " + "must be `NoResamplingStrategyTypes`, but got {}".format( + self.datamanager.resampling_strategy + )) + + self.splits = self.datamanager.splits + self.Y_target: Optional[np.ndarray] = None + self.Y_train_targets: np.ndarray = np.ones(self.y_train.shape) * np.NaN + self.pipeline: Optional[BaseEstimator] = None + + self.logger.debug("Search space updates :{}".format(self.search_space_updates)) + self.keep_models = keep_models + + def fit_predict_and_loss(self) -> None: + """Fit, predict and compute the loss for no resampling strategy""" + assert self.splits is not None, "Can't fit pipeline in {} is datamanager.splits is None" \ + .format(self.__class__.__name__) + additional_run_info: Optional[Dict] = None + split_id = 0 + self.logger.info("Starting fit {}".format(split_id)) + + pipeline = self._get_pipeline() + + train_split, test_split = self.splits[split_id] + assert test_split is None + self.Y_actual_train = self.y_train[train_split] + y_train_pred, y_valid_pred, y_test_pred = self._fit_and_predict(pipeline, split_id, + train_indices=train_split, + test_indices=test_split, + add_pipeline_to_self=True) + train_loss = self._loss(self.y_train[train_split], y_train_pred) + if y_valid_pred is not None: + loss = self._loss(self.y_valid, y_valid_pred) + elif y_test_pred is not None: + loss = self._loss(self.y_test, y_test_pred) + else: + loss = train_loss + + additional_run_info = pipeline.get_additional_run_info() if hasattr( + pipeline, 'get_additional_run_info') else {} + + status = StatusType.SUCCESS + + self.logger.debug("In train evaluator fit_predict_and_loss, num_run: {} loss:{}".format( + self.num_run, + loss + )) + self.finish_up( + loss=loss, + train_loss=train_loss, + valid_pred=y_valid_pred, + test_pred=y_test_pred, + additional_run_info=additional_run_info, + file_output=True, + status=status, + opt_pred=None + ) + + def _fit_and_predict(self, pipeline: BaseEstimator, fold: int, train_indices: Union[np.ndarray, List], + test_indices: None, + add_pipeline_to_self: bool + ) -> Tuple[np.ndarray, Optional[np.ndarray], Optional[np.ndarray]]: + + X = {'train_indices': train_indices, + 'val_indices': test_indices, + 'split_id': fold, + 'num_run': self.num_run, + **self.fit_dictionary} # fit dictionary + y = None + fit_and_suppress_warnings(self.logger, pipeline, X, y) + self.logger.info("Model fitted, now predicting") + ( + Y_train_pred, + Y_valid_pred, + Y_test_pred + ) = self._predict( + pipeline, + train_indices=train_indices, + ) + + if add_pipeline_to_self: + self.pipeline = pipeline + + return Y_train_pred, Y_valid_pred, Y_test_pred + + def _predict(self, pipeline: BaseEstimator, + train_indices: Union[np.ndarray, List] + ) -> Tuple[np.ndarray, Optional[np.ndarray], Optional[np.ndarray]]: + + train_pred = self.predict_function(subsampler(self.X_train, train_indices), pipeline, + self.y_train[train_indices]) + + if self.X_valid is not None: + valid_pred = self.predict_function(self.X_valid, pipeline, + self.y_valid) + else: + valid_pred = None + + if self.X_test is not None: + test_pred = self.predict_function(self.X_test, pipeline, + self.y_train[train_indices]) + else: + test_pred = None + + return train_pred, valid_pred, test_pred + + def finish_up(self, loss: Dict[str, float], train_loss: Dict[str, float], + valid_pred: Optional[np.ndarray], + test_pred: Optional[np.ndarray], additional_run_info: Optional[Dict], + file_output: bool, status: StatusType, + opt_pred: Optional[np.ndarray] + ) -> Optional[Tuple[float, float, int, Dict]]: + """This function does everything necessary after the fitting is done: + + * predicting + * saving the necessary files + We use it as the signal handler so we can recycle the code for the + normal usecase and when the runsolver kills us here :)""" + + self.duration = time.time() - self.starttime + + if file_output: + loss_, additional_run_info_ = self.file_output( + None, valid_pred, test_pred, + ) + else: + loss_ = None + additional_run_info_ = {} + + validation_loss, test_loss = self.calculate_auxiliary_losses( + valid_pred, test_pred + ) + + if loss_ is not None: + return self.duration, loss_, self.seed, additional_run_info_ + + cost = loss[self.metric.name] + + additional_run_info = ( + {} if additional_run_info is None else additional_run_info + ) + for metric_name, value in loss.items(): + additional_run_info[metric_name] = value + additional_run_info['duration'] = self.duration + additional_run_info['num_run'] = self.num_run + if train_loss is not None: + additional_run_info['train_loss'] = train_loss + if validation_loss is not None: + additional_run_info['validation_loss'] = validation_loss + if test_loss is not None: + additional_run_info['test_loss'] = test_loss + + rval_dict = {'loss': cost, + 'additional_run_info': additional_run_info, + 'status': status} + + self.queue.put(rval_dict) + return None + + def file_output( + self, + Y_optimization_pred: np.ndarray, + Y_valid_pred: np.ndarray, + Y_test_pred: np.ndarray, + ) -> Tuple[Optional[float], Dict]: + + # Abort if predictions contain NaNs + for y, s in [ + [Y_valid_pred, 'validation'], + [Y_test_pred, 'test'] + ]: + if y is not None and not np.all(np.isfinite(y)): + return ( + 1.0, + { + 'error': + 'Model predictions for %s set contains NaNs.' % s + }, + ) + + # Abort if we don't want to output anything. + if hasattr(self, 'disable_file_output'): + if self.disable_file_output: + return None, {} + else: + self.disabled_file_outputs = [] + + if hasattr(self, 'pipeline') and self.pipeline is not None: + if 'pipeline' not in self.disabled_file_outputs: + pipeline = self.pipeline + else: + pipeline = None + else: + pipeline = None + + self.logger.debug("Saving model {}_{}_{} to disk".format(self.seed, self.num_run, self.budget)) + self.backend.save_numrun_to_dir( + seed=int(self.seed), + idx=int(self.num_run), + budget=float(self.budget), + model=pipeline, + cv_model=None, + ensemble_predictions=None, + valid_predictions=( + Y_valid_pred if 'y_valid' not in + self.disabled_file_outputs else None + ), + test_predictions=( + Y_test_pred if 'y_test' not in + self.disabled_file_outputs else None + ), + ) + + return None, {} + + +# create closure for evaluating an algorithm +def eval_function( + backend: Backend, + queue: Queue, + metric: autoPyTorchMetric, + budget: float, + config: Optional[Configuration], + seed: int, + num_run: int, + include: Optional[Dict[str, Any]], + exclude: Optional[Dict[str, Any]], + disable_file_output: Union[bool, List], + output_y_hat_optimization: bool = False, + pipeline_config: Optional[Dict[str, Any]] = None, + budget_type: str = None, + init_params: Optional[Dict[str, Any]] = None, + logger_port: Optional[int] = None, + all_supported_metrics: bool = True, + search_space_updates: Optional[HyperparameterSearchSpaceUpdates] = None, + instance: str = None, +) -> None: + evaluator = FitEvaluator( + backend=backend, + queue=queue, + metric=metric, + configuration=config, + seed=seed, + num_run=num_run, + output_y_hat_optimization=output_y_hat_optimization, + include=include, + exclude=exclude, + disable_file_output=disable_file_output, + init_params=init_params, + budget=budget, + budget_type=budget_type, + logger_port=logger_port, + all_supported_metrics=all_supported_metrics, + pipeline_config=pipeline_config, + search_space_updates=search_space_updates + ) + evaluator.fit_predict_and_loss() diff --git a/autoPyTorch/evaluation/tae.py b/autoPyTorch/evaluation/tae.py index b1650113c..ec3ad038f 100644 --- a/autoPyTorch/evaluation/tae.py +++ b/autoPyTorch/evaluation/tae.py @@ -222,12 +222,26 @@ def __init__( self.search_space_updates = search_space_updates +<<<<<<< HEAD def _check_and_get_default_budget(self) -> float: budget_type_choices_tabular = ('epochs', 'runtime') budget_choices = { budget_type: float(self.pipeline_config.get(budget_type, np.inf)) for budget_type in budget_type_choices_tabular } +======= + if isinstance(self.resampling_strategy, (HoldoutValTypes, CrossValTypes)): + eval_function = autoPyTorch.evaluation.train_evaluator.eval_function + elif isinstance(self.resampling_strategy, NoResamplingStrategyTypes): + eval_function = autoPyTorch.evaluation.fit_evaluator.eval_function + else: + raise ValueError("resampling strategy must be in " + "(HoldoutValTypes, CrossValTypes, NoResamplingStrategyTypes), " + "but got {}.".format(self.resampling_strategy) + ) + + self.worst_possible_result = cost_for_crash +>>>>>>> Cocktail hotfixes (#245) budget_choices_forecasting = {budget_type: 1.0 for budget_type in FORECASTING_BUDGET_TYPE} budget_choices.update(budget_choices_forecasting) @@ -370,6 +384,7 @@ def run( info: Optional[List[RunValue]] additional_run_info: Dict[str, Any] try: + # By default, self.ta is fit_predict_try_except_decorator obj = pynisher.enforce_limits(**pynisher_arguments)(self.ta) obj(**obj_kwargs) except Exception as e: diff --git a/autoPyTorch/evaluation/train_evaluator.py b/autoPyTorch/evaluation/train_evaluator.py index ec870cdb3..867106804 100644 --- a/autoPyTorch/evaluation/train_evaluator.py +++ b/autoPyTorch/evaluation/train_evaluator.py @@ -152,11 +152,13 @@ def __init__(self, backend: Backend, queue: Queue, pipeline_config=pipeline_config, search_space_updates=search_space_updates ) - assert isinstance(self.datamanager.resampling_strategy, (CrossValTypes, HoldoutValTypes)),\ - "This Evaluator is used for HPO Search. " \ - "Val Split is required for HPO search. " \ - "Expected 'self.resampling_strategy' in" \ - " '(CrossValTypes, HoldoutValTypes)' got {}".format(self.datamanager.resampling_strategy) + + if not isinstance(self.datamanager.resampling_strategy, (CrossValTypes, HoldoutValTypes)): + raise ValueError( + 'TrainEvaluator expect to have (CrossValTypes, HoldoutValTypes) as ' + 'resampling_strategy, but got {}'.format(self.datamanager.resampling_strategy) + ) + if not isinstance(self.resampling_strategy, (CrossValTypes, HoldoutValTypes)): raise ValueError( diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py index 0e128d859..5388bfcc4 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py @@ -64,7 +64,8 @@ def _add_group(self, in_features: int, out_features: int, out_features (int): output dimensionality for the current block blocks_per_group (int): Number of ResNet per group last_block_index (int): block index for shake regularization - dropout (bool): whether or not use dropout + dropout (None, float): dropout value for the group. If none, + no dropout is applied. """ blocks = list() for i in range(blocks_per_group): @@ -180,9 +181,7 @@ def get_hyperparameter_search_space( if skip_connection_flag: - shake_drop_prob_flag = False - if 'shake-drop' in multi_branch_choice.value_range: - shake_drop_prob_flag = True + shake_drop_prob_flag = 'shake-drop' in multi_branch_choice.value_range mb_choice = get_hyperparameter(multi_branch_choice, CategoricalHyperparameter) cs.add_hyperparameter(mb_choice) @@ -290,13 +289,21 @@ def _build_block(self, in_features: int, out_features: int) -> nn.Module: if self.config['use_batch_norm']: layers.append(nn.BatchNorm1d(in_features)) layers.append(self.activation()) + elif not self.config['use_skip_connection']: + # if start norm is not None and skip connection is False + # we will never apply the start_norm for the first layer in the block, + # which is why we should account for this case. + if self.config['use_batch_norm']: + layers.append(nn.BatchNorm1d(in_features)) + layers.append(self.activation()) + layers.append(nn.Linear(in_features, out_features)) if self.config['use_batch_norm']: layers.append(nn.BatchNorm1d(out_features)) layers.append(self.activation()) - if self.config["use_dropout"]: + if self.dropout is not None: layers.append(nn.Dropout(self.dropout)) layers.append(nn.Linear(out_features, out_features)) @@ -321,6 +328,7 @@ def forward(self, x: torch.FloatTensor) -> torch.FloatTensor: if self.config["use_skip_connection"]: residual = self.shortcut(x) + # TODO make the below code better if self.config["use_skip_connection"]: if self.config["multi_branch_choice"] == 'shake-shake': x1 = self.layers(x) diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py index 1143177b8..a9e1f011e 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py @@ -31,11 +31,13 @@ def build_backbone(self, input_shape: Tuple[int, ...]) -> torch.nn.Sequential: out_features = self.config["output_dim"] # use the get_shaped_neuron_counts to update the number of units - neuron_counts = get_shaped_neuron_counts(self.config['resnet_shape'], - in_features, - out_features, - self.config['max_units'], - self.config['num_groups'] + 2)[:-1] + neuron_counts = get_shaped_neuron_counts( + shape=self.config['resnet_shape'], + in_feat=in_features, + out_feat=out_features, + max_neurons=self.config['max_units'], + layer_count=self.config['num_groups'] + 2, + )[:-1] self.config.update( {"num_units_%d" % (i): num for i, num in enumerate(neuron_counts)} ) @@ -45,12 +47,12 @@ def build_backbone(self, input_shape: Tuple[int, ...]) -> torch.nn.Sequential: # n_units for the architecture, since, it is mostly implemented for the # output layer, which is part of the head and not of the backbone. dropout_shape = get_shaped_neuron_counts( - self.config['dropout_shape'], 0, 0, 1000, self.config['num_groups'] - ) - - dropout_shape = [ - dropout / 1000 * self.config["max_dropout"] for dropout in dropout_shape - ] + shape=self.config['dropout_shape'], + in_feat=0, + out_feat=0, + max_neurons=self.config["max_dropout"], + layer_count=self.config['num_groups'] + 1, + )[:-1] self.config.update( {"dropout_%d" % (i + 1): dropout for i, dropout in enumerate(dropout_shape)} diff --git a/autoPyTorch/pipeline/components/setup/network_head/no_head.py b/autoPyTorch/pipeline/components/setup/network_head/no_head.py index f5cadb416..870f680fb 100644 --- a/autoPyTorch/pipeline/components/setup/network_head/no_head.py +++ b/autoPyTorch/pipeline/components/setup/network_head/no_head.py @@ -20,7 +20,7 @@ class NoHead(NetworkHeadComponent): """ def build_head(self, input_shape: Tuple[int, ...], output_shape: Tuple[int, ...]) -> nn.Module: - layers = [nn.Flatten()] + layers = [] in_features = np.prod(input_shape).item() out_features = np.prod(output_shape).item() layers.append(_activations[self.config["activation"]]()) @@ -34,8 +34,8 @@ def get_properties(dataset_properties: Optional[Dict[str, Any]] = None) -> Dict[ 'shortname': 'NoHead', 'name': 'NoHead', 'handles_tabular': True, - 'handles_image': True, - 'handles_time_series': True, + 'handles_image': False, + 'handles_time_series': False, } @staticmethod diff --git a/test/test_pipeline/components/setup/test_setup.py b/test/test_pipeline/components/setup/test_setup.py index e4b8deeb4..aafc34eb8 100644 --- a/test/test_pipeline/components/setup/test_setup.py +++ b/test/test_pipeline/components/setup/test_setup.py @@ -501,8 +501,7 @@ def test_dropout(self, resnet_shape): class TestNetworkHead: def test_all_heads_available(self): network_head_choice = NetworkHeadChoice(dataset_properties={}) - - assert len(network_head_choice.get_components().keys()) == 2 + assert len(network_head_choice.get_components().keys()) == 3 @pytest.mark.parametrize('task_type_input_output_shape', [(constants.IMAGE_CLASSIFICATION, (3, 64, 64), (5,)), (constants.IMAGE_REGRESSION, (3, 64, 64), (1,)), @@ -518,7 +517,9 @@ def test_dummy_forward_backward_pass(self, task_type_input_output_shape): if task_type in constants.CLASSIFICATION_TASKS: dataset_properties["num_classes"] = output_shape[0] - cs = network_head_choice.get_hyperparameter_search_space(dataset_properties=dataset_properties) + cs = network_head_choice.get_hyperparameter_search_space( + dataset_properties=dataset_properties, + ) # test 10 random configurations for _ in range(10): config = cs.sample_configuration() From c1a73f849183d5b62156ea09875cfb8d9ad3ae40 Mon Sep 17 00:00:00 2001 From: nabenabe0928 Date: Mon, 13 Sep 2021 10:52:22 +0200 Subject: [PATCH 32/50] [refactor] Address Shuhei's comments [fix] Fix Flake8 issues [refactor] Address Shuhei's comment [refactor] Address Shuhei's comments [refactor] Address Shuhei's comments [refactor] Address Shuhei's comments --- autoPyTorch/evaluation/tae.py | 14 ---- autoPyTorch/evaluation/train_evaluator.py | 4 - .../training/trainer/AdversarialTrainer.py | 23 +++--- .../training/trainer/GridCutMixTrainer.py | 18 +++-- .../training/trainer/RowCutMixTrainer.py | 28 ++++--- .../training/trainer/RowCutOutTrainer.py | 23 +++--- .../components/training/trainer/__init__.py | 12 +-- .../training/trainer/base_trainer.py | 45 ++++++----- .../training/trainer/cutout_utils.py | 78 +++++++++---------- .../training/trainer/mixup_utils.py | 2 +- .../components/training/trainer/utils.py | 9 ++- 11 files changed, 130 insertions(+), 126 deletions(-) diff --git a/autoPyTorch/evaluation/tae.py b/autoPyTorch/evaluation/tae.py index ec3ad038f..0307cab1b 100644 --- a/autoPyTorch/evaluation/tae.py +++ b/autoPyTorch/evaluation/tae.py @@ -222,26 +222,12 @@ def __init__( self.search_space_updates = search_space_updates -<<<<<<< HEAD def _check_and_get_default_budget(self) -> float: budget_type_choices_tabular = ('epochs', 'runtime') budget_choices = { budget_type: float(self.pipeline_config.get(budget_type, np.inf)) for budget_type in budget_type_choices_tabular } -======= - if isinstance(self.resampling_strategy, (HoldoutValTypes, CrossValTypes)): - eval_function = autoPyTorch.evaluation.train_evaluator.eval_function - elif isinstance(self.resampling_strategy, NoResamplingStrategyTypes): - eval_function = autoPyTorch.evaluation.fit_evaluator.eval_function - else: - raise ValueError("resampling strategy must be in " - "(HoldoutValTypes, CrossValTypes, NoResamplingStrategyTypes), " - "but got {}.".format(self.resampling_strategy) - ) - - self.worst_possible_result = cost_for_crash ->>>>>>> Cocktail hotfixes (#245) budget_choices_forecasting = {budget_type: 1.0 for budget_type in FORECASTING_BUDGET_TYPE} budget_choices.update(budget_choices_forecasting) diff --git a/autoPyTorch/evaluation/train_evaluator.py b/autoPyTorch/evaluation/train_evaluator.py index 867106804..9b823f350 100644 --- a/autoPyTorch/evaluation/train_evaluator.py +++ b/autoPyTorch/evaluation/train_evaluator.py @@ -415,11 +415,7 @@ def _predict(self, pipeline: BaseEstimator, # create closure for evaluating an algorithm -<<<<<<< HEAD def eval_train_function( -======= -def eval_function( ->>>>>>> Create fit evaluator, no resampling strategy and fix bug for test statistics backend: Backend, queue: Queue, metric: autoPyTorchMetric, diff --git a/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py b/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py index af000115f..f05908491 100644 --- a/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py @@ -37,7 +37,11 @@ def __init__( Args: epsilon (float): The perturbation magnitude. - + + References: + Explaining and Harnessing Adversarial Examples + Ian J. Goodfellow et. al. + https://arxiv.org/pdf/1412.6572.pdf """ super().__init__(random_state=random_state, weighted_loss=weighted_loss, @@ -96,10 +100,10 @@ def train_step(self, data: np.ndarray, targets: np.ndarray) -> Tuple[float, torc # training self.optimizer.zero_grad() original_outputs = self.model(original_data) - adversarial_output = self.model(adversarial_data) + adversarial_outputs = self.model(adversarial_data) loss_func = self.criterion_preparation(**criterion_kwargs) - loss = loss_func(self.criterion, original_outputs, adversarial_output) + loss = loss_func(self.criterion, original_outputs, adversarial_outputs) loss.backward() self.optimizer.step() if self.scheduler: @@ -125,6 +129,9 @@ def fgsm_attack( Returns: adv_data (np.ndarray): the adversarial examples. + + References: + https://pytorch.org/tutorials/beginner/fgsm_tutorial.html#fgsm-attack """ data_copy = deepcopy(data) data_copy = data_copy.float().to(self.device) @@ -159,7 +166,7 @@ def get_hyperparameter_search_space( dataset_properties: Optional[Dict] = None, weighted_loss: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="weighted_loss", - value_range=[True, False], + value_range=(True, False), default_value=True), la_steps: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="la_steps", @@ -196,9 +203,7 @@ def get_hyperparameter_search_space( add_hyperparameter(cs, epsilon, UniformFloatHyperparameter) add_hyperparameter(cs, use_stochastic_weight_averaging, CategoricalHyperparameter) - snapshot_ensemble_flag = False - if any(use_snapshot_ensemble.value_range): - snapshot_ensemble_flag = True + snapshot_ensemble_flag = any(use_snapshot_ensemble.value_range) use_snapshot_ensemble = get_hyperparameter(use_snapshot_ensemble, CategoricalHyperparameter) cs.add_hyperparameter(use_snapshot_ensemble) @@ -209,9 +214,7 @@ def get_hyperparameter_search_space( cond = EqualsCondition(se_lastk, use_snapshot_ensemble, True) cs.add_condition(cond) - lookahead_flag = False - if any(use_lookahead_optimizer.value_range): - lookahead_flag = True + lookahead_flag = any(use_lookahead_optimizer.value_range) use_lookahead_optimizer = get_hyperparameter(use_lookahead_optimizer, CategoricalHyperparameter) cs.add_hyperparameter(use_lookahead_optimizer) diff --git a/autoPyTorch/pipeline/components/training/trainer/GridCutMixTrainer.py b/autoPyTorch/pipeline/components/training/trainer/GridCutMixTrainer.py index 71a170c61..1fca0f93f 100644 --- a/autoPyTorch/pipeline/components/training/trainer/GridCutMixTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/GridCutMixTrainer.py @@ -26,14 +26,15 @@ def data_preparation(self, X: np.ndarray, y: np.ndarray, np.ndarray: that processes data typing.Dict[str, np.ndarray]: arguments to the criterion function """ - beta = 1.0 - lam = self.random_state.beta(beta, beta) - batch_size, channel, W, H = X.size() - index = torch.randperm(batch_size).cuda() if X.is_cuda else torch.randperm(batch_size) + alpha, beta = 1.0, 1.0 + lam = self.random_state.beta(alpha, beta) + batch_size, _, W, H = X.shape + device = torch.device('cuda' if X.is_cuda else 'cpu') + batch_indices = torch.randperm(batch_size).to(device) r = self.random_state.rand(1) if beta <= 0 or r > self.alpha: - return X, {'y_a': y, 'y_b': y[index], 'lam': 1} + return X, {'y_a': y, 'y_b': y[batch_indices], 'lam': 1} # Draw parameters of a random bounding box # Where to cut basically @@ -47,12 +48,13 @@ def data_preparation(self, X: np.ndarray, y: np.ndarray, bbx2 = np.clip(cx + cut_w // 2, 0, W) bby2 = np.clip(cy + cut_h // 2, 0, H) - X[:, :, bbx1:bbx2, bby1:bby2] = X[index, :, bbx1:bbx2, bby1:bby2] + X[:, :, bbx1:bbx2, bby1:bby2] = X[batch_indices, :, bbx1:bbx2, bby1:bby2] # Adjust lam - lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (X.size()[-1] * X.size()[-2])) + pixel_size = W * H + lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / pixel_size) - y_a, y_b = y, y[index] + y_a, y_b = y, y[batch_indices] return X, {'y_a': y_a, 'y_b': y_b, 'lam': lam} diff --git a/autoPyTorch/pipeline/components/training/trainer/RowCutMixTrainer.py b/autoPyTorch/pipeline/components/training/trainer/RowCutMixTrainer.py index 20d02c793..7c7dc6570 100644 --- a/autoPyTorch/pipeline/components/training/trainer/RowCutMixTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/RowCutMixTrainer.py @@ -26,25 +26,31 @@ def data_preparation(self, X: np.ndarray, y: np.ndarray, np.ndarray: that processes data typing.Dict[str, np.ndarray]: arguments to the criterion function """ - beta = 1.0 - lam = self.random_state.beta(beta, beta) - batch_size = X.size()[0] - index = torch.randperm(batch_size).cuda() if X.is_cuda else torch.randperm(batch_size) + alpha, beta = 1.0, 1.0 + lam = self.random_state.beta(alpha, beta) + batch_size = X.shape[0] + device = torch.device('cuda' if X.is_cuda else 'cpu') + batch_indices = torch.randperm(batch_size).to(device) r = self.random_state.rand(1) if beta <= 0 or r > self.alpha: - return X, {'y_a': y, 'y_b': y[index], 'lam': 1} + return X, {'y_a': y, 'y_b': y[batch_indices], 'lam': 1} - size = X.shape[1] - indices = torch.tensor(self.random_state.choice(range(1, size), max(1, np.int32(size * lam)), - replace=False)) + row_size = X.shape[1] + row_indices = torch.tensor( + self.random_state.choice( + range(1, row_size), + max(1, int(row_size * lam)), + replace=False + ) + ) - X[:, indices] = X[index, :][:, indices] + X[:, row_indices] = X[batch_indices, :][:, row_indices] # Adjust lam - lam = 1 - ((len(indices)) / (X.size()[1])) + lam = 1 - len(row_indices) / X.shape[1] - y_a, y_b = y, y[index] + y_a, y_b = y, y[batch_indices] return X, {'y_a': y_a, 'y_b': y_b, 'lam': lam} diff --git a/autoPyTorch/pipeline/components/training/trainer/RowCutOutTrainer.py b/autoPyTorch/pipeline/components/training/trainer/RowCutOutTrainer.py index 97f0caa18..02280f73d 100644 --- a/autoPyTorch/pipeline/components/training/trainer/RowCutOutTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/RowCutOutTrainer.py @@ -9,7 +9,9 @@ class RowCutOutTrainer(CutOut, BaseTrainerComponent): + # 0 is non-informative in image data NUMERICAL_VALUE = 0 + # -1 is the conceptually equivalent to 0 in a image, i.e. 0-pad CATEGORICAL_VALUE = -1 def data_preparation(self, X: np.ndarray, y: np.ndarray, @@ -36,23 +38,18 @@ def data_preparation(self, X: np.ndarray, y: np.ndarray, lam = 1 return X, {'y_a': y_a, 'y_b': y_b, 'lam': lam} - size = X.shape[1] - indices = self.random_state.choice(range(1, size), max(1, np.int32(size * self.patch_ratio)), - replace=False) + row_size = X.shape[1] + row_indices = self.random_state.choice(range(1, row_size), max(1, int(row_size * self.patch_ratio)), + replace=False) if not isinstance(self.numerical_columns, typing.Iterable): - raise ValueError("{} requires numerical columns information of {}" - "to prepare data got {}.".format(self.__class__.__name__, - typing.Iterable, - self.numerical_columns)) + raise ValueError("numerical_columns in {} must be iterable, " + "but got {}.".format(self.__class__.__name__, + self.numerical_columns)) + numerical_indices = torch.tensor(self.numerical_columns) - categorical_indices = torch.tensor([index for index in indices if index not in self.numerical_columns]) + categorical_indices = torch.tensor([idx for idx in row_indices if idx not in self.numerical_columns]) - # We use an ordinal encoder on the categorical columns of tabular data - # -1 is the conceptual equivalent to 0 in a image, that does not - # have color as a feature and hence the network has to learn to deal - # without this data. For numerical columns we use 0 to cutout the features - # similar to the effect that setting 0 as a pixel value in an image. X[:, categorical_indices.long()] = self.CATEGORICAL_VALUE X[:, numerical_indices.long()] = self.NUMERICAL_VALUE diff --git a/autoPyTorch/pipeline/components/training/trainer/__init__.py b/autoPyTorch/pipeline/components/training/trainer/__init__.py index cd53ebcaa..1ff296855 100755 --- a/autoPyTorch/pipeline/components/training/trainer/__init__.py +++ b/autoPyTorch/pipeline/components/training/trainer/__init__.py @@ -443,12 +443,13 @@ def _fit(self, X: Dict[str, Any], y: Any = None, **kwargs: Any) -> 'TrainerChoic if self.choice.use_stochastic_weight_averaging and self.choice.swa_updated: # update batch norm statistics - swa_utils.update_bn(X['train_data_loader'], self.choice.swa_model.double()) + swa_utils.update_bn(loader=X['train_data_loader'], model=self.choice.swa_model.double()) + # change model update_model_state_dict_from_swa(X['network'], self.choice.swa_model.state_dict()) if self.choice.use_snapshot_ensemble: for model in self.choice.model_snapshots: - swa_utils.update_bn(X['train_data_loader'], model.double()) + swa_utils.update_bn(loader=X['train_data_loader'], model=model.double()) # wrap up -- add score if not evaluating every epoch if not self.eval_valid_each_epoch(X): @@ -674,11 +675,12 @@ def __str__(self) -> str: def _get_search_space_updates(self, prefix: Optional[str] = None) -> Dict[str, HyperparameterSearchSpace]: """Get the search space updates with the given prefix - Keyword Arguments: - prefix {str} -- Only return search space updates with given prefix (default: {None}) + Args: + prefix (Optional[str]): Only return search space updates with given prefix Returns: - dict -- Mapping of search space updates. Keys don't contain the prefix. + Dict[str, HyperparameterSearchSpace]: + Mapping of search space updates. Keys don't contain the prefix. """ updates = super()._get_search_space_updates(prefix=prefix) diff --git a/autoPyTorch/pipeline/components/training/trainer/base_trainer.py b/autoPyTorch/pipeline/components/training/trainer/base_trainer.py index 145deed00..9e39b63cd 100644 --- a/autoPyTorch/pipeline/components/training/trainer/base_trainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/base_trainer.py @@ -31,7 +31,7 @@ ) from autoPyTorch.pipeline.components.training.trainer.utils import Lookahead from autoPyTorch.pipeline.components.training.metrics.utils import calculate_score -from autoPyTorch.pipeline.components.training.trainer.utils import Lookahead, swa_average_function +from autoPyTorch.pipeline.components.training.trainer.utils import Lookahead, swa_update from autoPyTorch.utils.common import FitRequirement, HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter from autoPyTorch.utils.implementations import get_loss_weight_strategy @@ -233,7 +233,7 @@ def __init__(self, weighted_loss: bool = False, use_snapshot_ensemble: bool = True, se_lastk: int = 3, use_lookahead_optimizer: bool = True, - random_state: Optional[Union[np.random.RandomState, int]] = None, + random_state: Optional[np.random.RandomState] = None, swa_model: Optional[torch.nn.Module] = None, model_snapshots: Optional[List[torch.nn.Module]] = None, **lookahead_config: Any) -> None: @@ -295,13 +295,14 @@ def prepare( # in case we are using swa, maintain an averaged model, if self.use_stochastic_weight_averaging: - self.swa_model = swa_utils.AveragedModel(self.model, avg_fn=swa_average_function) + self.swa_model = swa_utils.AveragedModel(self.model, avg_fn=swa_update) # in case we are using se or swa, initialise budget_threshold to know when to start swa or se self._budget_threshold = 0 if self.use_stochastic_weight_averaging or self.use_snapshot_ensemble: - assert budget_tracker.max_epochs is not None, "Can only use stochastic weight averaging or snapshot " \ - "ensemble when budget is epochs" + if budget_tracker.max_epochs is None: + raise ValueError("Budget for stochastic weight averaging or snapshot ensemble must be `epoch`.") + self._budget_threshold = int(0.75 * budget_tracker.max_epochs) # in case we are using se, initialise list to store model snapshots @@ -600,7 +601,7 @@ def get_hyperparameter_search_space( dataset_properties: Optional[Dict] = None, weighted_loss: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="weighted_loss", - value_range=[True, False], + value_range=(True, False), default_value=True), la_steps: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="la_steps", @@ -632,22 +633,30 @@ def get_hyperparameter_search_space( cs = ConfigurationSpace() add_hyperparameter(cs, use_stochastic_weight_averaging, CategoricalHyperparameter) + snapshot_ensemble_flag = any(use_snapshot_ensemble.value_range) + use_snapshot_ensemble = get_hyperparameter(use_snapshot_ensemble, CategoricalHyperparameter) - se_lastk = get_hyperparameter(se_lastk, Constant) - cs.add_hyperparameters([use_snapshot_ensemble, se_lastk]) - cond = EqualsCondition(se_lastk, use_snapshot_ensemble, True) - cs.add_condition(cond) + cs.add_hyperparameter(use_snapshot_ensemble) + if snapshot_ensemble_flag: + se_lastk = get_hyperparameter(se_lastk, Constant) + cs.add_hyperparameter(se_lastk) + cond = EqualsCondition(se_lastk, use_snapshot_ensemble, True) + cs.add_condition(cond) + + lookahead_flag = any(use_lookahead_optimizer.value_range) use_lookahead_optimizer = get_hyperparameter(use_lookahead_optimizer, CategoricalHyperparameter) cs.add_hyperparameter(use_lookahead_optimizer) - la_config_space = Lookahead.get_hyperparameter_search_space(la_steps=la_steps, - la_alpha=la_alpha) - parent_hyperparameter = {'parent': use_lookahead_optimizer, 'value': True} - cs.add_configuration_space( - Lookahead.__name__, - la_config_space, - parent_hyperparameter=parent_hyperparameter - ) + + if lookahead_flag: + la_config_space = Lookahead.get_hyperparameter_search_space(la_steps=la_steps, + la_alpha=la_alpha) + parent_hyperparameter = {'parent': use_lookahead_optimizer, 'value': True} + cs.add_configuration_space( + Lookahead.__name__, + la_config_space, + parent_hyperparameter=parent_hyperparameter + ) # TODO, decouple the weighted loss from the trainer if dataset_properties is not None: diff --git a/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py b/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py index 582014f9b..1b987d599 100644 --- a/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py +++ b/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py @@ -60,45 +60,45 @@ def criterion_preparation(self, y_a: np.ndarray, y_b: np.ndarray = None, lam: fl @staticmethod def get_hyperparameter_search_space( - dataset_properties: Optional[Dict] = None, - weighted_loss: HyperparameterSearchSpace = HyperparameterSearchSpace( - hyperparameter="weighted_loss", - value_range=[True, False], - default_value=True), - la_steps: HyperparameterSearchSpace = HyperparameterSearchSpace( - hyperparameter="la_steps", - value_range=(5, 10), - default_value=6, - log=False), - la_alpha: HyperparameterSearchSpace = HyperparameterSearchSpace( - hyperparameter="la_alpha", - value_range=(0.5, 0.8), - default_value=0.6, - log=False), - use_lookahead_optimizer: HyperparameterSearchSpace = HyperparameterSearchSpace( - hyperparameter="use_lookahead_optimizer", - value_range=(True, False), - default_value=True), - use_stochastic_weight_averaging: HyperparameterSearchSpace = HyperparameterSearchSpace( - hyperparameter="use_stochastic_weight_averaging", - value_range=(True, False), - default_value=True), - use_snapshot_ensemble: HyperparameterSearchSpace = HyperparameterSearchSpace( - hyperparameter="use_snapshot_ensemble", - value_range=(True, False), - default_value=True), - se_lastk: HyperparameterSearchSpace = HyperparameterSearchSpace( - hyperparameter="se_lastk", - value_range=(3,), - default_value=3), - patch_ratio: HyperparameterSearchSpace = HyperparameterSearchSpace( - hyperparameter="patch_ratio", - value_range=(0, 1), - default_value=0.2), - cutout_prob: HyperparameterSearchSpace = HyperparameterSearchSpace( - hyperparameter="cutout_prob", - value_range=(0, 1), - default_value=0.2), + dataset_properties: Optional[Dict] = None, + weighted_loss: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="weighted_loss", + value_range=(True, False), + default_value=True), + la_steps: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="la_steps", + value_range=(5, 10), + default_value=6, + log=False), + la_alpha: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="la_alpha", + value_range=(0.5, 0.8), + default_value=0.6, + log=False), + use_lookahead_optimizer: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="use_lookahead_optimizer", + value_range=(True, False), + default_value=True), + use_stochastic_weight_averaging: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="use_stochastic_weight_averaging", + value_range=(True, False), + default_value=True), + use_snapshot_ensemble: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="use_snapshot_ensemble", + value_range=(True, False), + default_value=True), + se_lastk: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="se_lastk", + value_range=(3,), + default_value=3), + patch_ratio: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="patch_ratio", + value_range=(0, 1), + default_value=0.2), + cutout_prob: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="cutout_prob", + value_range=(0, 1), + default_value=0.2), ) -> ConfigurationSpace: cs = ConfigurationSpace() diff --git a/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py b/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py index a0348a566..95e3ba8df 100644 --- a/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py +++ b/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py @@ -61,7 +61,7 @@ def get_hyperparameter_search_space( dataset_properties: Optional[Dict] = None, weighted_loss: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="weighted_loss", - value_range=[True, False], + value_range=(True, False), default_value=True), la_steps: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="la_steps", diff --git a/autoPyTorch/pipeline/components/training/trainer/utils.py b/autoPyTorch/pipeline/components/training/trainer/utils.py index 9193be6a6..cdc22402f 100644 --- a/autoPyTorch/pipeline/components/training/trainer/utils.py +++ b/autoPyTorch/pipeline/components/training/trainer/utils.py @@ -34,13 +34,16 @@ def update_model_state_dict_from_swa(model: torch.nn.Module, swa_state_dict: Dic model_state[name].copy_(param) -def swa_average_function(averaged_model_parameter: torch.nn.parameter.Parameter, - model_parameter: torch.nn.parameter.Parameter, - num_averaged: int) -> torch.nn.parameter.Parameter: +def swa_update(averaged_model_parameter: torch.nn.parameter.Parameter, + model_parameter: torch.nn.parameter.Parameter, + num_averaged: int) -> torch.nn.parameter.Parameter: """ Pickling the averaged function causes an error because of how pytorch initialises the average function. Passing this function fixes the issue. + The sequential update is performed via: + avg[n + 1] = (avg[n] * n + W[n + 1]) / (n + 1) + Args: averaged_model_parameter: model_parameter: From 769e0419aee96447bbd1a5e7a5b9db27c5ba14f8 Mon Sep 17 00:00:00 2001 From: nabenabe0928 Date: Wed, 15 Sep 2021 09:07:43 +0200 Subject: [PATCH 33/50] [doc] Add referencing to each regularization techniques --- .../training/trainer/AdversarialTrainer.py | 16 ++++++++-------- .../training/trainer/GridCutMixTrainer.py | 8 ++++++++ .../training/trainer/GridCutOutTrainer.py | 7 +++++++ .../training/trainer/RowCutOutTrainer.py | 8 ++++++++ .../components/training/trainer/mixup_utils.py | 7 +++++++ 5 files changed, 38 insertions(+), 8 deletions(-) diff --git a/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py b/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py index f05908491..157924ed0 100644 --- a/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py @@ -21,6 +21,13 @@ class AdversarialTrainer(BaseTrainerComponent): + """ + References: + Title: Explaining and Harnessing Adversarial Examples + Authors: Ian J. Goodfellow et. al. + URL: https://arxiv.org/pdf/1412.6572.pdf + Github URL: https://pytorch.org/tutorials/beginner/fgsm_tutorial.html#fgsm-attack + """ def __init__( self, epsilon: float, @@ -37,11 +44,7 @@ def __init__( Args: epsilon (float): The perturbation magnitude. - - References: - Explaining and Harnessing Adversarial Examples - Ian J. Goodfellow et. al. - https://arxiv.org/pdf/1412.6572.pdf + """ super().__init__(random_state=random_state, weighted_loss=weighted_loss, @@ -129,9 +132,6 @@ def fgsm_attack( Returns: adv_data (np.ndarray): the adversarial examples. - - References: - https://pytorch.org/tutorials/beginner/fgsm_tutorial.html#fgsm-attack """ data_copy = deepcopy(data) data_copy = data_copy.float().to(self.device) diff --git a/autoPyTorch/pipeline/components/training/trainer/GridCutMixTrainer.py b/autoPyTorch/pipeline/components/training/trainer/GridCutMixTrainer.py index 1fca0f93f..8b66b4fe8 100644 --- a/autoPyTorch/pipeline/components/training/trainer/GridCutMixTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/GridCutMixTrainer.py @@ -9,6 +9,14 @@ class GridCutMixTrainer(MixUp, BaseTrainerComponent): + """ # noqa + References: + Title: CutMix: Regularization Strategy to Train Strong Classifiers + with Localizable Features + Authors: Sangdoo Yun et. al. + URL: https://openaccess.thecvf.com/content_ICCV_2019/papers/Yun_CutMix_Regularization_Strategy_to_Train_Strong_Classifiers_With_Localizable_Features_ICCV_2019_paper.pdf + Github URL: https://github.com/clovaai/CutMix-PyTorch/blob/master/train.py#L227-L244 + """ def data_preparation(self, X: np.ndarray, y: np.ndarray, ) -> typing.Tuple[np.ndarray, typing.Dict[str, np.ndarray]]: diff --git a/autoPyTorch/pipeline/components/training/trainer/GridCutOutTrainer.py b/autoPyTorch/pipeline/components/training/trainer/GridCutOutTrainer.py index b2fd6151a..4d7f1099d 100644 --- a/autoPyTorch/pipeline/components/training/trainer/GridCutOutTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/GridCutOutTrainer.py @@ -7,6 +7,13 @@ class GridCutOutTrainer(CutOut, BaseTrainerComponent): + """ + References: + Title: Improved Regularization of Convolutional Neural Networks with Cutout + Authors: Terrance DeVries and Graham W. Taylor + URL: https://arxiv.org/pdf/1708.04552.pdf + Github URL: https://github.com/hysts/pytorch_cutout/blob/master/dataloader.py#L36-L68 + """ def data_preparation(self, X: np.ndarray, y: np.ndarray, ) -> typing.Tuple[np.ndarray, typing.Dict[str, np.ndarray]]: diff --git a/autoPyTorch/pipeline/components/training/trainer/RowCutOutTrainer.py b/autoPyTorch/pipeline/components/training/trainer/RowCutOutTrainer.py index 02280f73d..3e6f82953 100644 --- a/autoPyTorch/pipeline/components/training/trainer/RowCutOutTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/RowCutOutTrainer.py @@ -9,6 +9,14 @@ class RowCutOutTrainer(CutOut, BaseTrainerComponent): + """ + References: + Title: Improved Regularization of Convolutional Neural Networks with Cutout + Authors: Terrance DeVries and Graham W. Taylor + URL: https://arxiv.org/pdf/1708.04552.pdf + Github URL: https://github.com/hysts/pytorch_cutout/blob/master/dataloader.py#L36-L68 + """ + # 0 is non-informative in image data NUMERICAL_VALUE = 0 # -1 is the conceptually equivalent to 0 in a image, i.e. 0-pad diff --git a/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py b/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py index 95e3ba8df..e33011bf5 100644 --- a/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py +++ b/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py @@ -18,6 +18,13 @@ class MixUp: + """ + References: + Title: mixup: Beyond Empirical Risk Minimization + Authors: Hougyi Zhang et. al. + URL: https://arxiv.org/pdf/1710.09412.pdf%C2%A0 + Github URL: https://github.com/facebookresearch/mixup-cifar10/blob/master/train.py#L119-L138 + """ def __init__(self, alpha: float, weighted_loss: bool = False, random_state: Optional[np.random.RandomState] = None, From 0da4f72e320ac5d6a98678dd9d5bee0bd8b4a515 Mon Sep 17 00:00:00 2001 From: nabenabe0928 Date: Wed, 22 Sep 2021 04:22:43 +0900 Subject: [PATCH 34/50] [fix] Address Ravin's comments and fix range issues in row cut --- .../training/data_loader/base_data_loader.py | 4 ++-- .../training/trainer/GridCutMixTrainer.py | 8 ++++---- .../training/trainer/RowCutMixTrainer.py | 20 ++++++++++--------- .../training/trainer/RowCutOutTrainer.py | 7 ++++--- 4 files changed, 21 insertions(+), 18 deletions(-) diff --git a/autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py b/autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py index 769713680..a8651e158 100644 --- a/autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py +++ b/autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py @@ -64,10 +64,10 @@ def transform(self, X: Dict[str, Any]) -> Dict[str, Any]: underlying model and returns the transformed array. Args: - X (Dict[str, Any])): 'X' dictionary + X (Dict[str, Any])): fit dictionary Returns: - (Dict[str, Any]): the updated 'X' dictionary + (Dict[str, Any]): the updated fit dictionary """ X.update({'train_data_loader': self.train_data_loader, 'val_data_loader': self.val_data_loader, diff --git a/autoPyTorch/pipeline/components/training/trainer/GridCutMixTrainer.py b/autoPyTorch/pipeline/components/training/trainer/GridCutMixTrainer.py index 8b66b4fe8..24346042d 100644 --- a/autoPyTorch/pipeline/components/training/trainer/GridCutMixTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/GridCutMixTrainer.py @@ -38,11 +38,11 @@ def data_preparation(self, X: np.ndarray, y: np.ndarray, lam = self.random_state.beta(alpha, beta) batch_size, _, W, H = X.shape device = torch.device('cuda' if X.is_cuda else 'cpu') - batch_indices = torch.randperm(batch_size).to(device) + permed_indices = torch.randperm(batch_size).to(device) r = self.random_state.rand(1) if beta <= 0 or r > self.alpha: - return X, {'y_a': y, 'y_b': y[batch_indices], 'lam': 1} + return X, {'y_a': y, 'y_b': y[permed_indices], 'lam': 1} # Draw parameters of a random bounding box # Where to cut basically @@ -56,13 +56,13 @@ def data_preparation(self, X: np.ndarray, y: np.ndarray, bbx2 = np.clip(cx + cut_w // 2, 0, W) bby2 = np.clip(cy + cut_h // 2, 0, H) - X[:, :, bbx1:bbx2, bby1:bby2] = X[batch_indices, :, bbx1:bbx2, bby1:bby2] + X[:, :, bbx1:bbx2, bby1:bby2] = X[permed_indices, :, bbx1:bbx2, bby1:bby2] # Adjust lam pixel_size = W * H lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / pixel_size) - y_a, y_b = y, y[batch_indices] + y_a, y_b = y, y[permed_indices] return X, {'y_a': y_a, 'y_b': y_b, 'lam': lam} diff --git a/autoPyTorch/pipeline/components/training/trainer/RowCutMixTrainer.py b/autoPyTorch/pipeline/components/training/trainer/RowCutMixTrainer.py index 7c7dc6570..e36faf121 100644 --- a/autoPyTorch/pipeline/components/training/trainer/RowCutMixTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/RowCutMixTrainer.py @@ -30,27 +30,29 @@ def data_preparation(self, X: np.ndarray, y: np.ndarray, lam = self.random_state.beta(alpha, beta) batch_size = X.shape[0] device = torch.device('cuda' if X.is_cuda else 'cpu') - batch_indices = torch.randperm(batch_size).to(device) + permed_indices = torch.randperm(batch_size).to(device) r = self.random_state.rand(1) if beta <= 0 or r > self.alpha: - return X, {'y_a': y, 'y_b': y[batch_indices], 'lam': 1} + return X, {'y_a': y, 'y_b': y[permed_indices], 'lam': 1} - row_size = X.shape[1] - row_indices = torch.tensor( + # batch_size (permutation of rows), col_size = X.shape + col_size = X.shape[1] + col_indices = torch.tensor( self.random_state.choice( - range(1, row_size), - max(1, int(row_size * lam)), + range(col_size), + max(1, int(col_size * lam)), replace=False ) ) - X[:, row_indices] = X[batch_indices, :][:, row_indices] + # Replace selected columns with columns from another data point + X[:, col_indices] = X[permed_indices, :][:, col_indices] # Adjust lam - lam = 1 - len(row_indices) / X.shape[1] + lam = 1 - len(col_indices) / X.shape[1] - y_a, y_b = y, y[batch_indices] + y_a, y_b = y, y[permed_indices] return X, {'y_a': y_a, 'y_b': y_b, 'lam': lam} diff --git a/autoPyTorch/pipeline/components/training/trainer/RowCutOutTrainer.py b/autoPyTorch/pipeline/components/training/trainer/RowCutOutTrainer.py index 3e6f82953..5e1b9a1a3 100644 --- a/autoPyTorch/pipeline/components/training/trainer/RowCutOutTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/RowCutOutTrainer.py @@ -46,8 +46,9 @@ def data_preparation(self, X: np.ndarray, y: np.ndarray, lam = 1 return X, {'y_a': y_a, 'y_b': y_b, 'lam': lam} - row_size = X.shape[1] - row_indices = self.random_state.choice(range(1, row_size), max(1, int(row_size * self.patch_ratio)), + # (batch_size (permutation of rows), col_size) = X.shape + col_size = X.shape[1] + col_indices = self.random_state.choice(range(col_size), max(1, int(col_size * self.patch_ratio)), replace=False) if not isinstance(self.numerical_columns, typing.Iterable): @@ -56,7 +57,7 @@ def data_preparation(self, X: np.ndarray, y: np.ndarray, self.numerical_columns)) numerical_indices = torch.tensor(self.numerical_columns) - categorical_indices = torch.tensor([idx for idx in row_indices if idx not in self.numerical_columns]) + categorical_indices = torch.tensor([idx for idx in col_indices if idx not in self.numerical_columns]) X[:, categorical_indices.long()] = self.CATEGORICAL_VALUE X[:, numerical_indices.long()] = self.NUMERICAL_VALUE From c4a4565a92f445b57bf18fe5984854d98700c68e Mon Sep 17 00:00:00 2001 From: nabenabe0928 Date: Wed, 22 Sep 2021 05:01:06 +0900 Subject: [PATCH 35/50] [doc] Add the reference to the fit_dictionary --- autoPyTorch/evaluation/train_evaluator.py | 2 ++ .../components/preprocessing/tabular_preprocessing/utils.py | 4 ++++ autoPyTorch/utils/common.py | 2 ++ 3 files changed, 8 insertions(+) diff --git a/autoPyTorch/evaluation/train_evaluator.py b/autoPyTorch/evaluation/train_evaluator.py index 9b823f350..03ff69c32 100644 --- a/autoPyTorch/evaluation/train_evaluator.py +++ b/autoPyTorch/evaluation/train_evaluator.py @@ -362,6 +362,8 @@ def _fit_and_predict(self, pipeline: BaseEstimator, fold: int, train_indices: Un self.indices[fold] = ((train_indices, test_indices)) + # See autoPyTorch/pipeline/components/base_component.py::autoPyTorchComponent for more details + # about fit_dictionary X = {'train_indices': train_indices, 'val_indices': test_indices, 'split_id': fold, diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/utils.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/utils.py index f5af0a70b..e71583e3e 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/utils.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/utils.py @@ -12,8 +12,12 @@ def get_tabular_preprocessers(X: Dict[str, Any]) -> Dict[str, List[BaseEstimator Creates a dictionary with two keys, numerical- containing list of numerical preprocessors categorical- containing list of categorical preprocessors + Args: X: fit dictionary + See autoPyTorch/pipeline/components/base_component.py::autoPyTorchComponent for more details + about fit_dictionary + Returns: (Dict[str, List[BaseEstimator]]): dictionary with list of numerical and categorical preprocessors """ diff --git a/autoPyTorch/utils/common.py b/autoPyTorch/utils/common.py index 77f250164..a5082b1f3 100644 --- a/autoPyTorch/utils/common.py +++ b/autoPyTorch/utils/common.py @@ -168,6 +168,8 @@ def get_device_from_fit_dictionary(X: Dict[str, Any]) -> torch.device: Args: X (Dict[str, Any]): A fit dictionary to control how the pipeline is fitted + See autoPyTorch/pipeline/components/base_component.py::autoPyTorchComponent for more details + about fit_dictionary Returns: torch.device: Device to be used for training/inference From 654331662007f62b593979dfb4fe882ac66cae72 Mon Sep 17 00:00:00 2001 From: Arlind Kadra Date: Thu, 21 Oct 2021 22:46:16 +0200 Subject: [PATCH 36/50] Bug fixes (#249) * Update implementation * Coding style fixes * Implementation update * Style fix * Turn weighted loss into a constant again, implementation update * Cocktail branch inconsistencies (#275) * To nemo * Revert change in T_curr as results conclusively prove it should be 0 * Revert cutmix change after data from run * Final conclusion after results * FIX bug in shake alpha beta * Updated if is_training condition for shake drop * Remove temp fix in row cutmic * Cocktail fixes time debug (#286) * preprocess inside data validator * add time debug statements * Add fixes for categorical data * add fit_ensemble * add arlind fix for swa and se * fix bug in trainer choice fit * fix ensemble bug * Correct bug in cleanup * Cleanup for removing time debug statements * ablation for adversarial * shuffle false in dataloader * drop last false in dataloader * fix bug for validation set, and cutout and cutmix * shuffle = False * Shake Shake updates (#287) * To test locally * fix bug in trainer choice fit * fix ensemble bug * Correct bug in cleanup * To test locally * Cleanup for removing time debug statements * ablation for adversarial * shuffle false in dataloader * drop last false in dataloader * fix bug for validation set, and cutout and cutmix * To test locally * shuffle = False * To test locally * updates to search space * updates to search space * update branch with search space * undo search space update * fix bug in shake shake flag * limit to shake-even * restrict to even even * Add even even and others for shake-drop also * fix bug in passing alpha beta method * restrict to only even even * fix silly bug: * remove imputer and ordinal encoder for categorical transformer in feature validator * Address comments from shuhei * fix issues with ensemble fitting post hoc * Address comments on the PR * Fix flake and mypy errors * Address comments from PR #286 * fix bug in embedding * Update autoPyTorch/api/tabular_classification.py Co-authored-by: nabenabe0928 <47781922+nabenabe0928@users.noreply.github.com> * Update autoPyTorch/datasets/base_dataset.py Co-authored-by: nabenabe0928 <47781922+nabenabe0928@users.noreply.github.com> * Update autoPyTorch/datasets/base_dataset.py Co-authored-by: nabenabe0928 <47781922+nabenabe0928@users.noreply.github.com> * Update autoPyTorch/pipeline/components/training/trainer/base_trainer.py Co-authored-by: nabenabe0928 <47781922+nabenabe0928@users.noreply.github.com> * Address comments from shuhei * adress comments from shuhei * fix flake and mypy * Update autoPyTorch/pipeline/components/training/trainer/RowCutMixTrainer.py Co-authored-by: nabenabe0928 <47781922+nabenabe0928@users.noreply.github.com> * Update autoPyTorch/pipeline/tabular_classification.py Co-authored-by: nabenabe0928 <47781922+nabenabe0928@users.noreply.github.com> * Update autoPyTorch/pipeline/components/setup/network_backbone/utils.py Co-authored-by: nabenabe0928 <47781922+nabenabe0928@users.noreply.github.com> * Update autoPyTorch/pipeline/components/setup/network_backbone/utils.py Co-authored-by: nabenabe0928 <47781922+nabenabe0928@users.noreply.github.com> * Update autoPyTorch/pipeline/components/setup/network_backbone/utils.py Co-authored-by: nabenabe0928 <47781922+nabenabe0928@users.noreply.github.com> * Apply suggestions from code review Co-authored-by: nabenabe0928 <47781922+nabenabe0928@users.noreply.github.com> * increase threads_per_worker * fix bug in rowcutmix * Enhancement for the tabular validator. (#291) * Initial try at an enhancement for the tabular validator * Adding a few type annotations * Fixing bugs in implementation * Adding wrongly deleted code part during rebase * Fix bug in _get_args * Fix bug in _get_args * Addressing Shuhei's comments * Address Shuhei's comments * Refactoring code * Refactoring code * Typos fix and additional comments * Replace nan in categoricals with simple imputer * Remove unused function * add comment * Update autoPyTorch/data/tabular_feature_validator.py Co-authored-by: nabenabe0928 <47781922+nabenabe0928@users.noreply.github.com> * Update autoPyTorch/data/tabular_feature_validator.py Co-authored-by: nabenabe0928 <47781922+nabenabe0928@users.noreply.github.com> * Adding unit test for only nall columns in the tabular feature categorical evaluator * fix bug in remove all nan columns * Bug fix for making tests run by arlind * fix flake errors in feature validator * made typing code uniform * Apply suggestions from code review Co-authored-by: nabenabe0928 <47781922+nabenabe0928@users.noreply.github.com> * address comments from shuhei * address comments from shuhei (2) Co-authored-by: Ravin Kohli Co-authored-by: Ravin Kohli <13005107+ravinkohli@users.noreply.github.com> Co-authored-by: nabenabe0928 <47781922+nabenabe0928@users.noreply.github.com> * Apply suggestions from code review Co-authored-by: nabenabe0928 <47781922+nabenabe0928@users.noreply.github.com> * resolve code issues with new versions * Address comments from shuhei * make run_traditional_ml function * implement suggestion from shuhei and fix bug in rowcutmixtrainer * fix return type docstring * add better documentation and fix bug in shake_drop_get_bl * Apply suggestions from code review Co-authored-by: nabenabe0928 <47781922+nabenabe0928@users.noreply.github.com> * add test for comparator and other improvements based on PR comments * fix bug in test * [fix] Fix the condition in the raising error of all_nan_columns * [refactor] Unite name conventions of numpy array and pandas dataframe * [doc] Add the description about the tabular feature transformation * [doc] Add the description of the tabular feature transformation * address comments from arlind * address comments from arlind * change to as_tensor and address comments from arlind * correct description for functions in data module Co-authored-by: nabenabe0928 <47781922+nabenabe0928@users.noreply.github.com> Co-authored-by: Arlind Kadra Co-authored-by: nabenabe0928 * Addressing Shuhei's comments * flake8 problems fix * Update autoPyTorch/api/base_task.py Add indent. Co-authored-by: Ravin Kohli <13005107+ravinkohli@users.noreply.github.com> * Update autoPyTorch/api/base_task.py Add indent. Co-authored-by: Ravin Kohli <13005107+ravinkohli@users.noreply.github.com> * Update autoPyTorch/data/tabular_feature_validator.py Add indentation. Co-authored-by: Ravin Kohli <13005107+ravinkohli@users.noreply.github.com> * Update autoPyTorch/pipeline/components/setup/network_backbone/utils.py Add line indentation. Co-authored-by: Ravin Kohli <13005107+ravinkohli@users.noreply.github.com> * Update autoPyTorch/data/tabular_feature_validator.py Validate if there is a column transformer since for sparse matrices we will not have one. Co-authored-by: Ravin Kohli <13005107+ravinkohli@users.noreply.github.com> * Update autoPyTorch/utils/implementations.py Delete uncommented line. Co-authored-by: Ravin Kohli <13005107+ravinkohli@users.noreply.github.com> * Allow the number of threads to be given by the user * Removing unnecessary argument and refactoring the attribute. * Addressing Ravin's comments * Update autoPyTorch/pipeline/components/setup/network_backbone/utils.py Updating the function documentation according to the agreed style. Co-authored-by: Ravin Kohli <13005107+ravinkohli@users.noreply.github.com> * Update autoPyTorch/pipeline/components/setup/network_backbone/utils.py Providing information on the wrong method provided for shake-shake regularization. Co-authored-by: nabenabe0928 <47781922+nabenabe0928@users.noreply.github.com> * add todo for backend and accept changes from shuhei * Addressing Shuhei's and Ravin's comments * Addressing Shuhei's and Ravin's comments, bug fix * Update autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py Improving code readibility. Co-authored-by: nabenabe0928 <47781922+nabenabe0928@users.noreply.github.com> * Update autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py Improving consistency. Co-authored-by: nabenabe0928 <47781922+nabenabe0928@users.noreply.github.com> * bug fix Co-authored-by: Ravin Kohli <13005107+ravinkohli@users.noreply.github.com> Co-authored-by: nabenabe0928 <47781922+nabenabe0928@users.noreply.github.com> Co-authored-by: nabenabe0928 Co-authored-by: Ravin Kohli --- autoPyTorch/api/base_task.py | 338 ++++++++-- autoPyTorch/api/tabular_classification.py | 2 + autoPyTorch/api/tabular_regression.py | 3 +- autoPyTorch/data/base_feature_validator.py | 45 +- autoPyTorch/data/base_target_validator.py | 5 +- autoPyTorch/data/tabular_feature_validator.py | 334 +++++----- autoPyTorch/datasets/base_dataset.py | 1 + autoPyTorch/ensemble/singlebest_ensemble.py | 5 +- autoPyTorch/pipeline/base_pipeline.py | 48 +- .../TabularColumnTransformer.py | 5 +- .../encoding/base_encoder.py | 2 +- .../imputation/base_imputer.py | 2 +- .../scaling/base_scaler.py | 2 +- .../components/setup/network/base_network.py | 16 +- .../setup/network_backbone/ResNetBackbone.py | 91 +-- .../network_backbone/ShapedResNetBackbone.py | 22 +- .../setup/network_backbone/utils.py | 67 +- .../base_network_embedding.py | 7 +- .../components/setup/network_head/no_head.py | 1 - .../setup/optimizer/AdamWOptimizer.py | 4 +- .../training/data_loader/base_data_loader.py | 4 +- .../training/trainer/AdversarialTrainer.py | 27 +- .../training/trainer/RowCutMixTrainer.py | 43 +- .../training/trainer/RowCutOutTrainer.py | 41 +- .../training/trainer/StandardTrainer.py | 4 +- .../components/training/trainer/__init__.py | 6 +- .../training/trainer/base_trainer.py | 26 +- .../training/trainer/cutout_utils.py | 14 +- .../training/trainer/mixup_utils.py | 17 +- .../pipeline/tabular_classification.py | 13 + autoPyTorch/utils/backend.py | 575 ++++++++++++++++++ autoPyTorch/utils/common.py | 24 + .../example_custom_configuration_space.py | 141 +++++ .../example_posthoc_ensemble_fit.py | 81 +++ test/test_data/test_feature_validator.py | 166 ++++- 35 files changed, 1796 insertions(+), 386 deletions(-) create mode 100644 autoPyTorch/utils/backend.py create mode 100644 examples/40_advanced/40_advanced/example_custom_configuration_space.py create mode 100644 examples/40_advanced/40_advanced/example_posthoc_ensemble_fit.py diff --git a/autoPyTorch/api/base_task.py b/autoPyTorch/api/base_task.py index 83f3840ba..3a902878e 100644 --- a/autoPyTorch/api/base_task.py +++ b/autoPyTorch/api/base_task.py @@ -228,6 +228,9 @@ def __init__( self._logger: Optional[PicklableClientLogger] = None self.dataset_name: Optional[str] = None self.cv_models_: Dict = {} + self.precision: Optional[int] = None + self.opt_metric: Optional[str] = None + self.dataset: Optional[BaseDataset] = None self._results_manager = ResultsManager() @@ -585,6 +588,7 @@ def _clean_logger(self) -> None: self.logging_server.join(timeout=5) self.logging_server.terminate() del self.stop_logging_server + self._logger = None def _create_dask_client(self) -> None: """ @@ -600,7 +604,7 @@ def _create_dask_client(self) -> None: dask.distributed.LocalCluster( n_workers=self.n_jobs, processes=True, - threads_per_worker=1, + threads_per_worker=self.n_threads, # We use the temporal directory to save the # dask workers, because deleting workers # more time than deleting backend directories @@ -674,6 +678,23 @@ def _load_models(self) -> bool: return True + def _cleanup(self) -> None: + """ + Closes the different servers created during api search. + Returns: + None + """ + if hasattr(self, '_logger') and self._logger is not None: + self._logger.info("Closing the dask infrastructure") + self._close_dask_client() + self._logger.info("Finished closing the dask infrastructure") + + # Clean up the logger + self._logger.info("Starting to clean up the logger") + self._clean_logger() + else: + self._close_dask_client() + def _load_best_individual_model(self) -> SingleBest: """ In case of failure during ensemble building, @@ -914,6 +935,38 @@ def _do_traditional_prediction(self, time_left: int, func_eval_time_limit_secs: save_external=True) return + def run_traditional_ml( + self, + current_task_name: str, + runtime_limit: int, + func_eval_time_limit_secs: int + ) -> None: + """ + This function can be used to run the suite of traditional machine + learning models during the current task (for e.g, ensemble fit, search) + + Args: + current_task_name (str): name of the current task, + runtime_limit (int): time limit for fitting traditional models, + func_eval_time_limit_secs (int): Time limit + for a single call to the machine learning model. + Model fitting will be terminated if the machine + learning algorithm runs over the time limit. + """ + assert self._logger is not None # for mypy compliancy + if STRING_TO_TASK_TYPES[self.task_type] in REGRESSION_TASKS: + self._logger.warning("Traditional Pipeline is not enabled for regression. Skipping...") + else: + traditional_task_name = 'runTraditional' + self._stopwatch.start_task(traditional_task_name) + elapsed_time = self._stopwatch.wall_elapsed(current_task_name) + time_for_traditional = int(runtime_limit - elapsed_time) + self._do_traditional_prediction( + func_eval_time_limit_secs=func_eval_time_limit_secs, + time_left=time_for_traditional, + ) + self._stopwatch.stop_task(traditional_task_name) + def _search( self, optimize_metric: str, @@ -1076,8 +1129,10 @@ def _search( """ if self.task_type != dataset.task_type: raise ValueError("Incompatible dataset entered for current task," - "expected dataset to have task type :{} got " + "expected dataset to have task type :{} but got " ":{}".format(self.task_type, dataset.task_type)) + if precision not in [16, 32, 64]: + raise ValueError("precision must be one of 16, 32, 64 but got {}".format(precision)) # Initialise information needed for the experiment experiment_task_name: str = 'runSearch' @@ -1182,28 +1237,25 @@ def _search( ) # ============> Run dummy predictions - dummy_task_name = 'runDummy' - self._stopwatch.start_task(dummy_task_name) - self._do_dummy_prediction() - self._stopwatch.stop_task(dummy_task_name) + # We only want to run dummy predictions in case we want to build an ensemble + if self.ensemble_size > 0: + dummy_task_name = 'runDummy' + self._stopwatch.start_task(dummy_task_name) + self._do_dummy_prediction() + self._stopwatch.stop_task(dummy_task_name) # ============> Run traditional ml - - if enable_traditional_pipeline: - traditional_task_name = 'runTraditional' - self._stopwatch.start_task(traditional_task_name) - elapsed_time = self._stopwatch.wall_elapsed(self.dataset_name) - # We want time for at least 1 Neural network in SMAC - time_for_traditional = int( - self._time_for_task - elapsed_time - func_eval_time_limit_secs - ) - self._do_traditional_prediction( - func_eval_time_limit_secs=func_eval_time_limit_secs, - time_left=time_for_traditional, - ) - self._stopwatch.stop_task(traditional_task_name) + # We only want to run traditional predictions in case we want to build an ensemble + # We want time for at least 1 Neural network in SMAC + if enable_traditional_pipeline and self.ensemble_size > 0: + traditional_runtime_limit = int(self._time_for_task - func_eval_time_limit_secs) + self.run_traditional_ml(current_task_name=self.dataset_name, + runtime_limit=traditional_runtime_limit, + func_eval_time_limit_secs=func_eval_time_limit_secs) # ============> Starting ensemble + self.precision = precision + self.opt_metric = optimize_metric elapsed_time = self._stopwatch.wall_elapsed(self.dataset_name) time_left_for_ensembles = max(0, total_walltime_limit - elapsed_time) proc_ensemble = None @@ -1220,28 +1272,12 @@ def _search( self._logger.info("Starting ensemble") ensemble_task_name = 'ensemble' self._stopwatch.start_task(ensemble_task_name) - proc_ensemble = EnsembleBuilderManager( - start_time=time.time(), - time_left_for_ensembles=time_left_for_ensembles, - backend=copy.deepcopy(self._backend), - dataset_name=str(dataset.dataset_name), - output_type=STRING_TO_OUTPUT_TYPES[dataset.output_type], - task_type=STRING_TO_TASK_TYPES[self.task_type], - metrics=[self._metric], - opt_metric=optimize_metric, - ensemble_size=self.ensemble_size, - ensemble_nbest=self.ensemble_nbest, - max_models_on_disc=self.max_models_on_disc, - seed=self.seed, - max_iterations=None, - read_at_most=sys.maxsize, - ensemble_memory_limit=self._memory_limit, - random_state=self.seed, - precision=precision, - logger_port=self._logger_port, - pynisher_context=self._multiprocessing_context, - metrics_kwargs=self._metrics_kwargs, - ) + proc_ensemble = self._init_ensemble_builder(time_left_for_ensembles=time_left_for_ensembles, + ensemble_size=self.ensemble_size, + ensemble_nbest=self.ensemble_nbest, + precision=precision, + optimize_metric=self.opt_metric + ) self._stopwatch.stop_task(ensemble_task_name) # ==> Run SMAC @@ -1328,18 +1364,12 @@ def _search( pd.DataFrame(self.ensemble_performance_history).to_json( os.path.join(self._backend.internals_directory, 'ensemble_history.json')) - self._logger.info("Closing the dask infrastructure") - self._close_dask_client() - self._logger.info("Finished closing the dask infrastructure") - if load_models: self._logger.info("Loading models...") self._load_models() self._logger.info("Finished loading models...") - # Clean up the logger - self._logger.info("Starting to clean up the logger") - self._clean_logger() + self._cleanup() return self @@ -1611,7 +1641,7 @@ def fit_pipeline( exclude=self.exclude_components, search_space_updates=self.search_space_updates) dataset_properties = dataset.get_dataset_properties(dataset_requirements) - self._backend.save_datamanager(dataset) + self._backend.replace_datamanager(dataset) if self._logger is None: self._logger = self._get_logger(dataset.dataset_name) @@ -1724,6 +1754,207 @@ def _get_fitted_pipeline( budget=float(run_info.budget), ) + def fit_ensemble( + self, + optimize_metric: Optional[str] = None, + precision: Optional[int] = None, + ensemble_nbest: int = 50, + ensemble_size: int = 50, + load_models: bool = True, + time_for_task: int = 100, + func_eval_time_limit_secs: int = 50, + enable_traditional_pipeline: bool = True, + ) -> 'BaseTask': + """ + Enables post-hoc fitting of the ensemble after the `search()` + method is finished. This method creates an ensemble using all + the models stored on disk during the smbo run. + + Args: + optimize_metric (str): name of the metric that is used to + evaluate a pipeline. if not specified, value passed to search will be used + precision (int), (default=32): Numeric precision used when loading + ensemble data. Can be either 16, 32 or 64. + ensemble_nbest (Optional[int]): + only consider the ensemble_nbest models to build the ensemble. + If None, uses the value stored in class attribute `ensemble_nbest`. + ensemble_size (int) (default=50): + Number of models added to the ensemble built by + Ensemble selection from libraries of models. + Models are drawn with replacement. + enable_traditional_pipeline (bool), (default=True): + We fit traditional machine learning algorithms + (LightGBM, CatBoost, RandomForest, ExtraTrees, KNN, SVM) + prior building PyTorch Neural Networks. You can disable this + feature by turning this flag to False. All machine learning + algorithms that are fitted during search() are considered for + ensemble building. + load_models (bool), (default=True): Whether to load the + models after fitting AutoPyTorch. + time_for_task (int), (default=100): Time limit + in seconds for the search of appropriate models. + By increasing this value, autopytorch has a higher + chance of finding better models. + func_eval_time_limit_secs (int), (default=None): Time limit + for a single call to the machine learning model. + Model fitting will be terminated if the machine + learning algorithm runs over the time limit. Set + this value high enough so that typical machine + learning algorithms can be fit on the training + data. + When set to None, this time will automatically be set to + total_walltime_limit // 2 to allow enough time to fit + at least 2 individual machine learning algorithms. + Set to np.inf in case no time limit is desired. + + Returns: + self + """ + # Make sure that input is valid + if self.dataset is None or self.opt_metric is None: + raise ValueError("fit_ensemble() can only be called after `search()`. " + "Please call the `search()` method of {} prior to " + "fit_ensemble().".format(self.__class__.__name__)) + + if precision not in [16, 32, 64]: + raise ValueError("precision must be one of 16, 32, 64 but got {}".format(precision)) + + if self._logger is None: + self._logger = self._get_logger(self.dataset.dataset_name) + + # Create a client if needed + if self._dask_client is None: + self._create_dask_client() + else: + self._is_dask_client_internally_created = False + + ensemble_fit_task_name = 'EnsembleFit' + self._stopwatch.start_task(ensemble_fit_task_name) + if enable_traditional_pipeline: + if func_eval_time_limit_secs is None or func_eval_time_limit_secs > time_for_task: + self._logger.warning( + 'Time limit for a single run is higher than total time ' + 'limit. Capping the limit for a single run to the total ' + 'time given to Ensemble fit (%f)' % time_for_task + ) + func_eval_time_limit_secs = time_for_task + + # Make sure that at least 2 models are created for the ensemble process + num_models = time_for_task // func_eval_time_limit_secs + if num_models < 2: + func_eval_time_limit_secs = time_for_task // 2 + self._logger.warning( + "Capping the func_eval_time_limit_secs to {} to have " + "time for at least 2 models to ensemble.".format( + func_eval_time_limit_secs + ) + ) + # ============> Run Dummy predictions + dummy_task_name = 'runDummy' + self._stopwatch.start_task(dummy_task_name) + self._do_dummy_prediction() + self._stopwatch.stop_task(dummy_task_name) + + # ============> Run traditional ml + if enable_traditional_pipeline: + self.run_traditional_ml(current_task_name=ensemble_fit_task_name, + runtime_limit=time_for_task, + func_eval_time_limit_secs=func_eval_time_limit_secs) + + elapsed_time = self._stopwatch.wall_elapsed(ensemble_fit_task_name) + time_left_for_ensemble = int(time_for_task - elapsed_time) + manager = self._init_ensemble_builder( + time_left_for_ensembles=time_left_for_ensemble, + optimize_metric=self.opt_metric if optimize_metric is None else optimize_metric, + precision=self.precision if precision is None else precision, + ensemble_size=ensemble_size, + ensemble_nbest=ensemble_nbest, + ) + + manager.build_ensemble(self._dask_client) + future = manager.futures.pop() + result = future.result() + if result is None: + raise ValueError("Errors occurred while building the ensemble - please" + " check the log file and command line output for error messages.") + self.ensemble_performance_history, _, _, _ = result + + if load_models: + self._load_models() + + self._stopwatch.stop_task(ensemble_fit_task_name) + + self._cleanup() + + return self + + def _init_ensemble_builder( + self, + time_left_for_ensembles: float, + optimize_metric: str, + ensemble_nbest: int, + ensemble_size: int, + precision: int = 32, + ) -> EnsembleBuilderManager: + """ + Initializes an `EnsembleBuilderManager`. + Args: + time_left_for_ensembles (float): + Time (in seconds) allocated to building the ensemble + optimize_metric (str): + Name of the metric to optimize the ensemble. + ensemble_nbest (int): + only consider the ensemble_nbest models to build the ensemble. + ensemble_size (int): + Number of models added to the ensemble built by + Ensemble selection from libraries of models. + Models are drawn with replacement. + precision (int), (default=32): Numeric precision used when loading + ensemble data. Can be either 16, 32 or 64. + + Returns: + EnsembleBuilderManager + """ + if self._logger is None: + raise ValueError("logger should be initialized to fit ensemble") + if self.dataset is None: + raise ValueError("ensemble can only be initialised after or during `search()`. " + "Please call the `search()` method of {}.".format(self.__class__.__name__)) + + self._logger.info("Starting ensemble") + ensemble_task_name = 'ensemble' + self._stopwatch.start_task(ensemble_task_name) + + # Use the current thread to start the ensemble builder process + # The function ensemble_builder_process will internally create a ensemble + # builder in the provide dask client + required_dataset_properties = {'task_type': self.task_type, + 'output_type': self.dataset.output_type} + proc_ensemble = EnsembleBuilderManager( + start_time=time.time(), + time_left_for_ensembles=time_left_for_ensembles, + backend=copy.deepcopy(self._backend), + dataset_name=str(self.dataset.dataset_name), + output_type=STRING_TO_OUTPUT_TYPES[self.dataset.output_type], + task_type=STRING_TO_TASK_TYPES[self.task_type], + metrics=[self._metric] if self._metric is not None else get_metrics( + dataset_properties=required_dataset_properties, names=[optimize_metric]), + opt_metric=optimize_metric, + ensemble_size=ensemble_size, + ensemble_nbest=ensemble_nbest, + max_models_on_disc=self.max_models_on_disc, + seed=self.seed, + max_iterations=None, + read_at_most=sys.maxsize, + ensemble_memory_limit=self._memory_limit, + random_state=self.seed, + precision=precision, + logger_port=self._logger_port, + ) + self._stopwatch.stop_task(ensemble_task_name) + + return proc_ensemble + def predict( self, X_test: np.ndarray, @@ -1775,7 +2006,7 @@ def predict( predictions = self.ensemble_.predict(all_predictions) - self._clean_logger() + self._cleanup() return predictions @@ -1815,10 +2046,7 @@ def __getstate__(self) -> Dict[str, Any]: return self.__dict__ def __del__(self) -> None: - # Clean up the logger - self._clean_logger() - - self._close_dask_client() + self._cleanup() # When a multiprocessing work is done, the # objects are deleted. We don't want to delete run areas diff --git a/autoPyTorch/api/tabular_classification.py b/autoPyTorch/api/tabular_classification.py index facb59f99..ed920960b 100644 --- a/autoPyTorch/api/tabular_classification.py +++ b/autoPyTorch/api/tabular_classification.py @@ -447,6 +447,8 @@ def search( dataset_compression=self._dataset_compression, feat_types=feat_types) + if self.dataset is None: + raise ValueError("`dataset` in {} must be initialized, but got None".format(self.__class__.__name__)) return self._search( dataset=self.dataset, optimize_metric=optimize_metric, diff --git a/autoPyTorch/api/tabular_regression.py b/autoPyTorch/api/tabular_regression.py index e0c1e4eac..d14e6891c 100644 --- a/autoPyTorch/api/tabular_regression.py +++ b/autoPyTorch/api/tabular_regression.py @@ -79,7 +79,6 @@ class TabularRegressionTask(BaseTask): Search space updates that can be used to modify the search space of particular components or choice modules of the pipeline """ - def __init__( self, seed: int = 1, @@ -448,6 +447,8 @@ def search( dataset_compression=self._dataset_compression, feat_types=feat_types) + if self.dataset is None: + raise ValueError("`dataset` in {} must be initialized, but got None".format(self.__class__.__name__)) return self._search( dataset=self.dataset, optimize_metric=optimize_metric, diff --git a/autoPyTorch/data/base_feature_validator.py b/autoPyTorch/data/base_feature_validator.py index c7facd997..38d5e0ef7 100644 --- a/autoPyTorch/data/base_feature_validator.py +++ b/autoPyTorch/data/base_feature_validator.py @@ -1,5 +1,5 @@ import logging -from typing import List, Optional, Union +from typing import List, Optional, Set, Tuple, Union import numpy as np @@ -24,16 +24,14 @@ class BaseFeatureValidator(BaseEstimator): List of the column types found by this estimator during fit. data_type (str): Class name of the data type provided during fit. - column_transformer (Optional[BaseEstimator]) + encoder (Optional[BaseEstimator]) Host a encoder object if the data requires transformation (for example, - if provided a categorical column in a pandas DataFrame) - transformed_columns (List[str]) - List of columns that were encoded. + if provided a categorical column in a pandas DataFrame). """ def __init__( self, logger: Optional[Union[PicklableClientLogger, logging.Logger]] = None, - ): + ) -> None: # Register types to detect unsupported data format changes self.feat_types: Optional[List[str]] = None self.data_type: Optional[type] = None @@ -41,7 +39,6 @@ def __init__( self.column_order: List[str] = [] self.column_transformer: Optional[BaseEstimator] = None - self.transformed_columns: List[str] = [] self.logger: Union[ PicklableClientLogger, logging.Logger @@ -53,6 +50,8 @@ def __init__( self.categorical_columns: List[int] = [] self.numerical_columns: List[int] = [] + self.all_nan_columns: Optional[Set[Union[int, str]]] = None + self._is_fitted = False def fit( @@ -75,7 +74,7 @@ def fit( # If a list was provided, it will be converted to pandas if isinstance(X_train, list): - X_train, X_test = self.list_to_dataframe(X_train, X_test) + X_train, X_test = self.list_to_pandas(X_train, X_test) self._check_data(X_train) @@ -109,6 +108,7 @@ def _fit( self: The fitted base estimator """ + raise NotImplementedError() def _check_data( @@ -118,11 +118,12 @@ def _check_data( """ Feature dimensionality and data type checks - Arguments: + Args: X (SUPPORTED_FEAT_TYPES): A set of features that are going to be validated (type and dimensionality checks) and a encoder fitted in the case the data needs encoding """ + raise NotImplementedError() def transform( @@ -139,4 +140,30 @@ def transform( np.ndarray: The transformed array """ + + raise NotImplementedError() + + def list_to_pandas( + self, + X_train: SUPPORTED_FEAT_TYPES, + X_test: Optional[SUPPORTED_FEAT_TYPES] = None, + ) -> Tuple[pd.DataFrame, Optional[pd.DataFrame]]: + """ + Converts a list to a pandas DataFrame. In this process, column types are inferred. + + If test data is provided, we proactively match it to train data + + Args: + X_train (SUPPORTED_FEAT_TYPES): + A set of features that are going to be validated (type and dimensionality + checks) and a encoder fitted in the case the data needs encoding + X_test (Optional[SUPPORTED_FEAT_TYPES]): + A hold out set of data used for checking + Returns: + pd.DataFrame: + transformed train data from list to pandas DataFrame + pd.DataFrame: + transformed test data from list to pandas DataFrame + """ + raise NotImplementedError() diff --git a/autoPyTorch/data/base_target_validator.py b/autoPyTorch/data/base_target_validator.py index 9943d5c55..5c209b9a5 100644 --- a/autoPyTorch/data/base_target_validator.py +++ b/autoPyTorch/data/base_target_validator.py @@ -36,7 +36,7 @@ def __init__(self, logging.Logger ] ] = None, - ): + ) -> None: self.is_classification = is_classification self.data_type: Optional[type] = None @@ -86,6 +86,7 @@ def fit( np.shape(y_test) )) if isinstance(y_train, pd.DataFrame): + y_train = cast(pd.DataFrame, y_train) y_test = cast(pd.DataFrame, y_test) if y_train.columns.tolist() != y_test.columns.tolist(): raise ValueError( @@ -131,7 +132,7 @@ def _fit( def transform( self, - y: Union[SupportedTargetTypes], + y: SupportedTargetTypes, ) -> np.ndarray: """ Args: diff --git a/autoPyTorch/data/tabular_feature_validator.py b/autoPyTorch/data/tabular_feature_validator.py index 5e6013e90..2b4285402 100644 --- a/autoPyTorch/data/tabular_feature_validator.py +++ b/autoPyTorch/data/tabular_feature_validator.py @@ -10,12 +10,12 @@ from scipy.sparse import issparse, spmatrix import sklearn.utils -from sklearn import preprocessing from sklearn.base import BaseEstimator from sklearn.compose import ColumnTransformer from sklearn.exceptions import NotFittedError from sklearn.impute import SimpleImputer from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import OneHotEncoder, StandardScaler from autoPyTorch.data.base_feature_validator import BaseFeatureValidator, SupportedFeatTypes from autoPyTorch.utils.common import ispandas @@ -24,6 +24,7 @@ def _create_column_transformer( preprocessors: Dict[str, List[BaseEstimator]], + numerical_columns: List[str], categorical_columns: List[str], ) -> ColumnTransformer: """ @@ -34,6 +35,8 @@ def _create_column_transformer( Args: preprocessors (Dict[str, List[BaseEstimator]]): Dictionary containing list of numerical and categorical preprocessors. + numerical_columns (List[str]): + List of names of numerical columns categorical_columns (List[str]): List of names of categorical columns @@ -41,11 +44,17 @@ def _create_column_transformer( ColumnTransformer """ - categorical_pipeline = make_pipeline(*preprocessors['categorical']) + numerical_pipeline = 'drop' + categorical_pipeline = 'drop' + if len(numerical_columns) > 0: + numerical_pipeline = make_pipeline(*preprocessors['numerical']) + if len(categorical_columns) > 0: + categorical_pipeline = make_pipeline(*preprocessors['categorical']) return ColumnTransformer([ - ('categorical_pipeline', categorical_pipeline, categorical_columns)], - remainder='passthrough' + ('categorical_pipeline', categorical_pipeline, categorical_columns), + ('numerical_pipeline', numerical_pipeline, numerical_columns)], + remainder='drop' ) @@ -60,11 +69,15 @@ def get_tabular_preprocessors() -> Dict[str, List[BaseEstimator]]: preprocessors: Dict[str, List[BaseEstimator]] = dict() # Categorical Preprocessors - onehot_encoder = preprocessing.OrdinalEncoder(handle_unknown='use_encoded_value', - unknown_value=-1) + onehot_encoder = OneHotEncoder(categories='auto', sparse=False, handle_unknown='ignore') categorical_imputer = SimpleImputer(strategy='constant', copy=False) + # Numerical Preprocessors + numerical_imputer = SimpleImputer(strategy='median', copy=False) + standard_scaler = StandardScaler(with_mean=True, with_std=True, copy=False) + preprocessors['categorical'] = [categorical_imputer, onehot_encoder] + preprocessors['numerical'] = [numerical_imputer, standard_scaler] return preprocessors @@ -126,7 +139,6 @@ def _comparator(cmp1: str, cmp2: str) -> int: if cmp1 not in choices or cmp2 not in choices: raise ValueError('The comparator for the column order only accepts {}, ' 'but got {} and {}'.format(choices, cmp1, cmp2)) - idx1, idx2 = choices.index(cmp1), choices.index(cmp2) return idx1 - idx2 @@ -152,65 +164,43 @@ def _fit( # The final output of a validator is a numpy array. But pandas # gives us information about the column dtype if isinstance(X, np.ndarray): - X = self.numpy_array_to_pandas(X) + + X = self.numpy_to_pandas(X) + # Replace the data type from the previously saved type. + self.data_type = type(X) + # save all the information about the column order and data types + self._check_data(X) if ispandas(X) and not issparse(X): X = cast(pd.DataFrame, X) - # Treat a column with all instances a NaN as numerical - # This will prevent doing encoding to a categorical column made completely - # out of nan values -- which will trigger a fail, as encoding is not supported - # with nan values. - # Columns that are completely made of NaN values are provided to the pipeline - # so that later stages decide how to handle them - - # Clear whatever null column markers we had previously - self.null_columns.clear() - if np.any(pd.isnull(X)): - for column in X.columns: - if X[column].isna().all(): - self.null_columns.add(column) - X[column] = pd.to_numeric(X[column]) - # Also note this change in self.dtypes - if len(self.dtypes) != 0: - self.dtypes[list(X.columns).index(column)] = X[column].dtype - - if not X.select_dtypes(include='object').empty: - X = self.infer_objects(X) + + self.all_nan_columns = set([column for column in X.columns if X[column].isna().all()]) self.transformed_columns, self.feat_types = self.get_columns_to_encode(X) assert self.feat_types is not None - if len(self.transformed_columns) > 0: - - preprocessors = get_tabular_preprocessors() - self.column_transformer = _create_column_transformer( - preprocessors=preprocessors, - categorical_columns=self.transformed_columns, - ) + preprocessors = get_tabular_preprocessors() + self.column_transformer = _create_column_transformer( + preprocessors=preprocessors, + categorical_columns=self.transformed_columns, + ) - # Mypy redefinition - assert self.column_transformer is not None - self.column_transformer.fit(X) + # Mypy redefinition + assert self.column_transformer is not None + self.column_transformer.fit(X) - # The column transformer reorders the feature types - # therefore, we need to change the order of columns as well - # This means categorical columns are shifted to the left - self.feat_types = sorted( - self.feat_types, - key=functools.cmp_to_key(self._comparator) - ) + # The column transformer reorders the feature types + # therefore, we need to change the order of columns as well + # This means categorical columns are shifted to the left - encoded_categories = self.column_transformer.\ - named_transformers_['categorical_pipeline'].\ - named_steps['ordinalencoder'].categories_ - self.categories = [ - # We fit an ordinal encoder, where all categorical - # columns are shifted to the left - list(range(len(cat))) - for cat in encoded_categories - ] + self.feat_types = sorted( + self.feat_types, + key=functools.cmp_to_key(self._comparator) + ) + # differently to categorical_columns and numerical_columns, + # this saves the index of the column. for i, type_ in enumerate(self.feat_types): if 'numerical' in type_: self.numerical_columns.append(i) @@ -219,6 +209,7 @@ def _fit( # Lastly, store the number of features self.num_features = np.shape(X)[1] + return self def transform( @@ -237,49 +228,70 @@ def transform( Return: np.ndarray: The transformed array + + Note: + The default transform performs the folloing: + * simple imputation for both + * scaling for numerical + * one-hot encoding for categorical + For example, here is a simple case + of which all the columns are categorical. + data = [ + {'A': 1, 'B': np.nan, 'C': np.nan}, + {'A': np.nan, 'B': 3, 'C': np.nan}, + {'A': 2, 'B': np.nan, 'C': np.nan} + ] + and suppose all the columns are categorical, + then + * `A` in {np.nan, 1, 2} + * `B` in {np.nan, 3} + * `C` in {np.nan} <=== it will be dropped. + + So in the column A, + * np.nan ==> [1, 0, 0] (always the index 0) + * 1 ==> [0, 1, 0] + * 2 ==> [0, 0, 1] + in the column B, + * np.nan ==> [1, 0] + * 3 ==> [0, 1] + Therefore, by concatenating, + * {'A': 1, 'B': np.nan, 'C': np.nan} ==> [0, 1, 0, 1, 0] + * {'A': np.nan, 'B': 3, 'C': np.nan} ==> [1, 0, 0, 0, 1] + * {'A': 2, 'B': np.nan, 'C': np.nan} ==> [0, 0, 1, 1, 0] + ==> [ + [0, 1, 0, 1, 0], + [1, 0, 0, 0, 1], + [0, 0, 1, 1, 0] + ] """ if not self._is_fitted: raise NotFittedError("Cannot call transform on a validator that is not fitted") # If a list was provided, it will be converted to pandas if isinstance(X, list): - X, _ = self.list_to_dataframe(X) + X, _ = self.list_to_pandas(X) if isinstance(X, np.ndarray): - X = self.numpy_array_to_pandas(X) + X = self.numpy_to_pandas(X) if hasattr(X, "iloc") and not issparse(X): X = cast(pd.DataFrame, X) - # If we had null columns in our fit call and we made them numeric, then: - # - If the columns are null even in transform, apply the same procedure. - # - Otherwise, substitute the values with np.NaN and then make the columns numeric. - # If the column is null here, but it was not in fit, it does not matter. - for column in self.null_columns: - # The column is not null, make it null since it was null in fit. - if not X[column].isna().all(): - X[column] = np.NaN - X[column] = pd.to_numeric(X[column]) - - # for the test set, if we have columns with only null values - # they will probably have a numeric type. If these columns were not - # with only null values in the train set, they should be converted - # to the type that they had during fitting. - for column in X.columns: - if X[column].isna().all(): - X[column] = X[column].astype(self.dtypes[list(X.columns).index(column)]) - - # Also remove the object dtype for new data - if not X.select_dtypes(include='object').empty: - X = self.infer_objects(X) # Check the data here so we catch problems on new test data self._check_data(X) - # We also need to fillna on the transformation - # in case test data is provided - X = self.impute_nan_in_categories(X) - if self.encoder is not None: - X = self.encoder.transform(X) + # in case of test data being all none and train data + # having a value for a categorical column. + # We need to convert the column in test data to + # object otherwise the test column is interpreted as float + if len(self.categorical_columns) > 0: + categorical_columns = self.column_transformer.transformers_[0][-1] + for column in categorical_columns: + if X[column].isna().all(): + X[column] = X[column].astype('object') + + if self.column_transformer is not None: + X = self.column_transformer.transform(X) # Sparse related transformations # Not all sparse format support index sorting @@ -349,7 +361,8 @@ def _check_data( X = cast(pd.DataFrame, X) # Handle objects if possible - if not X.select_dtypes(include='object').empty: + exist_object_columns = has_object_columns(X.dtypes.values) + if exist_object_columns: X = self.infer_objects(X) # Define the column to be encoded here as the feature validator is fitted once @@ -359,25 +372,26 @@ def _check_data( column_order = [column for column in X.columns] if len(self.column_order) > 0: if self.column_order != column_order: - raise ValueError("Changing the column order of the features after fit() is " - "not supported. Fit() method was called with " - "{} whereas the new features have {} as type".format(self.column_order, - column_order,) - ) + raise ValueError("The column order of the features must not be changed after fit(), but" + " the column order are different between training ({}) and" + " test ({}) datasets.".format(self.column_order, column_order)) else: self.column_order = column_order dtypes = [dtype.name for dtype in X.dtypes] - if len(self.dtypes) > 0: - if self.dtypes != dtypes: - raise ValueError("Changing the dtype of the features after fit() is " - "not supported. Fit() method was called with " - "{} whereas the new features have {} as type".format(self.dtypes, - dtypes, - ) - ) - else: + dtypes_diff = [s_dtype != dtype for s_dtype, dtype in zip(self.dtypes, dtypes)] + if len(self.dtypes) == 0: self.dtypes = dtypes + elif ( + any(dtypes_diff) # the dtypes of some columns are different in train and test dataset + and self.all_nan_columns is not None # Ignore all_nan_columns is None + and len(set(X.columns[dtypes_diff]).difference(self.all_nan_columns)) != 0 + ): + # The dtypes can be different if and only if the column belongs + # to all_nan_columns as these columns would be imputed. + raise ValueError("The dtype of the features must not be changed after fit(), but" + " the dtypes of some columns are different between training ({}) and" + " test ({}) datasets.".format(self.dtypes, dtypes)) def get_columns_to_encode( self, @@ -438,7 +452,7 @@ def _validate_feat_types(self, X: pd.DataFrame) -> None: def _get_columns_to_encode( self, X: pd.DataFrame, - ) -> Tuple[List[str], List[str]]: + ) -> Tuple[List[str], List[str], List[str]]: """ Return the columns to be transformed as well as the type of feature for each column from a pandas dataframe. @@ -452,9 +466,11 @@ def _get_columns_to_encode( checks) and an encoder fitted in the case the data needs encoding Returns: - transformed_columns (List[str]): - Columns to encode, if any - feat_type: + categorical_columns (List[str]) + List of the names of categorical columns. + numerical_columns (List[str]) + List of the names of numerical columns. + feat_type (List[str]) Type of each column numerical/categorical """ @@ -462,63 +478,57 @@ def _get_columns_to_encode( return self.transformed_columns, self.feat_types # Register if a column needs encoding - transformed_columns = [] - + categorical_columns = [] # Also, register the feature types for the estimator feat_types = [] # Make sure each column is a valid type for column in X.columns: - if X[column].dtype.name in ['category', 'bool']: + if self.all_nan_columns is not None and column in self.all_nan_columns: + continue + column_dtype = self.dtypes[i] + err_msg = "Valid types are `numerical`, `categorical` or `boolean`, " \ + "but input column {} has an invalid type `{}`.".format(column, column_dtype) + if column_dtype in ['category', 'bool']: - transformed_columns.append(column) if self.feat_types is not None and self.feat_types[i].lower() == 'numerical': raise ValueError(f"Passed numerical as the feature type for column: {column} " f"but the column is categorical") feat_types.append('categorical') + categorical_columns.append(column) + # Move away from np.issubdtype as it causes # TypeError: data type not understood in certain pandas types - elif not is_numeric_dtype(X[column]): - if X[column].dtype.name == 'object': - raise ValueError( - "Input Column {} has invalid type object. " - "Cast it to a valid dtype before using it in AutoPyTorch. " - "Valid types are numerical, categorical or boolean. " - "You can cast it to a valid dtype using " - "pandas.Series.astype ." - "If working with string objects, the following " - "tutorial illustrates how to work with text data: " - "https://scikit-learn.org/stable/tutorial/text_analytics/working_with_text_data.html".format( - # noqa: E501 - column, - ) - ) - elif pd.core.dtypes.common.is_datetime_or_timedelta_dtype( - X[column].dtype - ): - raise ValueError( - "AutoPyTorch does not support time and/or date datatype as given " - "in column {}. Please convert the time information to a numerical value " - "first. One example on how to do this can be found on " - "https://stats.stackexchange.com/questions/311494/".format( - column, - ) - ) - else: - raise ValueError( - "Input Column {} has unsupported dtype {}. " - "Supported column types are categorical/bool/numerical dtypes. " - "Make sure your data is formatted in a correct way, " - "before feeding it to AutoPyTorch.".format( - column, - X[column].dtype.name, - ) + elif is_numeric_dtype(column_dtype): + feat_types.append('numerical') + elif column_dtype == 'object': + # TODO verify how would this happen when we always convert the object dtypes to category + raise TypeError( + "{} Cast it to a valid dtype before feeding it to AutoPyTorch. " + "You can cast it to a valid dtype using pandas.Series.astype." + "If you are working with string objects, the following " + "tutorial illustrates how to work with text data: " + "https://scikit-learn.org/stable/tutorial/text_analytics/working_with_text_data.html".format( + # noqa: E501 + err_msg, ) + ) + elif pd.core.dtypes.common.is_datetime_or_timedelta_dtype(column_dtype): + raise TypeError( + "{} Convert the time information to a numerical value" + " before feeding it to AutoPyTorch. " + "One example of the conversion can be found on " + "https://stats.stackexchange.com/questions/311494/".format(err_msg) + ) else: - feat_types.append('numerical') - return transformed_columns, feat_types + raise TypeError( + "{} Make sure your data is formatted in a correct way" + "before feeding it to AutoPyTorch.".format(err_msg) + ) - def list_to_dataframe( + return categorical_columns, feat_types + + def list_to_pandas( self, X_train: SupportedFeatTypes, X_test: Optional[SupportedFeatTypes] = None, @@ -543,7 +553,7 @@ def list_to_dataframe( """ # If a list was provided, it will be converted to pandas - X_train = pd.DataFrame(data=X_train).infer_objects() + X_train = pd.DataFrame(data=X_train).convert_dtypes() self.logger.warning("The provided feature types to AutoPyTorch are of type list." "Features have been interpreted as: {}".format([(col, t) for col, t in zip(X_train.columns, X_train.dtypes)])) @@ -552,11 +562,12 @@ def list_to_dataframe( self.logger.warning("Train features are a list while the provided test data" "is {}. X_test will be casted as DataFrame.".format(type(X_test)) ) - X_test = pd.DataFrame(data=X_test).infer_objects() + X_test = pd.DataFrame(data=X_test).convert_dtypes() + return X_train, X_test - def numpy_array_to_pandas( - self, + @staticmethod + def numpy_to_pandas( X: np.ndarray, ) -> pd.DataFrame: """ @@ -595,10 +606,33 @@ def infer_objects(self, X: pd.DataFrame) -> pd.DataFrame: self.logger.warning(f'Casting the column {key} to {dtype} caused the exception {e}') pass else: + # Calling for the first time to infer the categories X = X.infer_objects() - for column in X.columns: - if not is_numeric_dtype(X[column]): + for column, data_type in zip(X.columns, X.dtypes): + if not is_numeric_dtype(data_type): X[column] = X[column].astype('category') - self.object_dtype_mapping = {column: X[column].dtype for column in X.columns} + + # only numerical attributes and categories + self.object_dtype_mapping = {column: data_type for column, data_type in zip(X.columns, X.dtypes)} + self.logger.debug(f"Infer Objects: {self.object_dtype_mapping}") + return X + + +def has_object_columns( + feature_types: pd.Series, +) -> bool: + """ + Indicate whether on a Series of dtypes for a Pandas DataFrame + there exists one or more object columns. + + Args: + feature_types (pd.Series): The feature types for a DataFrame. + + Returns: + bool: + True if the DataFrame dtypes contain an object column, False + otherwise. + """ + return np.dtype('O') in feature_types diff --git a/autoPyTorch/datasets/base_dataset.py b/autoPyTorch/datasets/base_dataset.py index 755bf1e18..a63e2b108 100644 --- a/autoPyTorch/datasets/base_dataset.py +++ b/autoPyTorch/datasets/base_dataset.py @@ -359,6 +359,7 @@ def get_dataset(self, split_id: int, train: bool) -> Dataset: train (bool): whether the dataset is required for training or evaluating. Returns: + Dataset: the reduced dataset to be used for testing """ # Subset creates a dataset. Splits is a (train_indices, test_indices) tuple diff --git a/autoPyTorch/ensemble/singlebest_ensemble.py b/autoPyTorch/ensemble/singlebest_ensemble.py index 9fcbeee82..890563c14 100644 --- a/autoPyTorch/ensemble/singlebest_ensemble.py +++ b/autoPyTorch/ensemble/singlebest_ensemble.py @@ -3,7 +3,7 @@ import numpy as np -from smac.runhistory.runhistory import RunHistory +from smac.runhistory.runhistory import RunHistory, StatusType from autoPyTorch.automl_common.common.utils.backend import Backend from autoPyTorch.ensemble.abstract_ensemble import AbstractEnsemble @@ -52,6 +52,9 @@ def get_identifiers_from_run_history(self) -> List[Tuple[int, int, float]]: for run_key in self.run_history.data.keys(): run_value = self.run_history.data[run_key] + if run_value.status == StatusType.CRASHED: + continue + score = self.metric._optimum - (self.metric._sign * run_value.cost) if (score > best_model_score and self.metric._sign > 0) \ diff --git a/autoPyTorch/pipeline/base_pipeline.py b/autoPyTorch/pipeline/base_pipeline.py index 6c6116a73..7c2efa798 100644 --- a/autoPyTorch/pipeline/base_pipeline.py +++ b/autoPyTorch/pipeline/base_pipeline.py @@ -22,7 +22,9 @@ get_match_array ) from autoPyTorch.utils.common import FitRequirement -from autoPyTorch.utils.hyperparameter_search_space_update import HyperparameterSearchSpaceUpdates +from autoPyTorch.utils.hyperparameter_search_space_update import ( + HyperparameterSearchSpaceUpdates +) PipelineStepType = Union[autoPyTorchComponent, autoPyTorchChoice] @@ -405,6 +407,7 @@ def _check_search_space_updates(self, include: Optional[Dict[str, Any]], raise ValueError("Unknown node name. Expected update node name to be in {} " "got {}".format(self.named_steps.keys(), update.node_name)) node = self.named_steps[update.node_name] + node_name = node.__class__.__name__ # if node is a choice module if hasattr(node, 'get_components'): split_hyperparameter = update.hyperparameter.split(':') @@ -446,14 +449,15 @@ def _check_search_space_updates(self, include: Optional[Dict[str, Any]], if choice in exclude[update.node_name]: raise ValueError("Found {} in exclude".format(choice)) if choice not in components.keys(): - raise ValueError("Unknown hyperparameter for choice {}. " + raise ValueError("Unknown component choice for node {}. " "Expected update hyperparameter " - "to be in {} got {}".format(node.__class__.__name__, - components.keys(), choice)) + "to be in {}, but got {}".format(node_name, + components.keys(), choice)) # check if the component whose hyperparameter # needs to be updated is in components of the # choice module elif split_hyperparameter[0] not in components.keys(): +<<<<<<< HEAD hp_in_component = False if hasattr(node, 'additional_components') and node.additional_components: # This is designed for forecasting network encoder: @@ -471,6 +475,12 @@ def _check_search_space_updates(self, include: Optional[Dict[str, Any]], "Expected update hyperparameter " "to be in {} got {}".format(node.__class__.__name__, components.keys(), split_hyperparameter[0])) +======= + raise ValueError("Unknown component choice for node {}. " + "Expected update component " + "to be in {}, but got {}".format(node_name, + components.keys(), split_hyperparameter[0])) +>>>>>>> Bug fixes (#249) else: # check if hyperparameter is in the search space of the component component = components[split_hyperparameter[0]] @@ -483,14 +493,16 @@ def _check_search_space_updates(self, include: Optional[Dict[str, Any]], component.get_hyperparameter_search_space( dataset_properties=self.dataset_properties).get_hyperparameter_names()]): continue - raise ValueError("Unknown hyperparameter for component {}. " - "Expected update hyperparameter " - "to be in {} got {}".format(node.__class__.__name__, - component. - get_hyperparameter_search_space( - dataset_properties=self.dataset_properties). - get_hyperparameter_names(), - split_hyperparameter[1])) + component_hyperparameters = component.get_hyperparameter_search_space( + dataset_properties=self.dataset_properties).get_hyperparameter_names() + raise ValueError("Unknown hyperparameter for component {} of node {}." + " Expected update hyperparameter " + "to be in {}, but got {}.".format(component.__name__, + node_name, + component_hyperparameters, + split_hyperparameter[1] + ) + ) else: if update.hyperparameter not in node.get_hyperparameter_search_space( dataset_properties=self.dataset_properties): @@ -498,13 +510,13 @@ def _check_search_space_updates(self, include: Optional[Dict[str, Any]], node.get_hyperparameter_search_space( dataset_properties=self.dataset_properties).get_hyperparameter_names()]): continue - raise ValueError("Unknown hyperparameter for component {}. " + node_hyperparameters = node.get_hyperparameter_search_space( + dataset_properties=self.dataset_properties).get_hyperparameter_names() + raise ValueError("Unknown hyperparameter for node {}. " "Expected update hyperparameter " - "to be in {} got {}".format(node.__class__.__name__, - node. - get_hyperparameter_search_space( - dataset_properties=self.dataset_properties). - get_hyperparameter_names(), update.hyperparameter)) + "to be in {}, but got {}".format(node_name, + node_hyperparameters, + update.hyperparameter)) def _get_pipeline_steps(self, dataset_properties: Optional[Dict[str, BaseDatasetPropertiesType]] ) -> List[Tuple[str, PipelineStepType]]: diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/TabularColumnTransformer.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/TabularColumnTransformer.py index 02a3085b0..b8805c809 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/TabularColumnTransformer.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/TabularColumnTransformer.py @@ -4,14 +4,14 @@ from sklearn.base import BaseEstimator from sklearn.compose import ColumnTransformer -from sklearn.pipeline import make_pipeline +# from sklearn.pipeline import make_pipeline import torch from autoPyTorch.pipeline.components.preprocessing.tabular_preprocessing.base_tabular_preprocessing import ( autoPyTorchTabularPreprocessingComponent ) -from autoPyTorch.pipeline.components.preprocessing.tabular_preprocessing.utils import get_tabular_preprocessers +# from autoPyTorch.pipeline.components.preprocessing.tabular_preprocessing.utils import get_tabular_preprocessers from autoPyTorch.utils.common import FitRequirement, subsampler @@ -48,6 +48,7 @@ def fit(self, X: Dict[str, Any], y: Any = None) -> "TabularColumnTransformer": Returns: "TabularColumnTransformer": an instance of self """ + self.check_requirements(X, y) preprocessors = get_tabular_preprocessers(X) diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/encoding/base_encoder.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/encoding/base_encoder.py index eadc0a188..9829cadcd 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/encoding/base_encoder.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/encoding/base_encoder.py @@ -28,5 +28,5 @@ def transform(self, X: Dict[str, Any]) -> Dict[str, Any]: if self.preprocessor['numerical'] is None and self.preprocessor['categorical'] is None: raise ValueError("cant call transform on {} without fitting first." .format(self.__class__.__name__)) - X.update({'encoder': self.preprocessor}) + # X.update({'encoder': self.preprocessor}) return X diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/imputation/base_imputer.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/imputation/base_imputer.py index 1f33a765a..9bab21122 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/imputation/base_imputer.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/imputation/base_imputer.py @@ -28,5 +28,5 @@ def transform(self, X: Dict[str, Any]) -> Dict[str, Any]: if self.preprocessor['numerical'] is None and len(X["dataset_properties"]["numerical_columns"]) != 0: raise ValueError("cant call transform on {} without fitting first." .format(self.__class__.__name__)) - X.update({'imputer': self.preprocessor}) + # X.update({'imputer': self.preprocessor}) return X diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/base_scaler.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/base_scaler.py index 39834dd2b..270fac246 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/base_scaler.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/base_scaler.py @@ -28,5 +28,5 @@ def transform(self, X: Dict[str, Any]) -> Dict[str, Any]: if self.preprocessor['numerical'] is None and self.preprocessor['categorical'] is None: raise ValueError("cant call transform on {} without fitting first." .format(self.__class__.__name__)) - X.update({'scaler': self.preprocessor}) + # X.update({'scaler': self.preprocessor}) return X diff --git a/autoPyTorch/pipeline/components/setup/network/base_network.py b/autoPyTorch/pipeline/components/setup/network/base_network.py index 8b75ab66a..02782e7a2 100644 --- a/autoPyTorch/pipeline/components/setup/network/base_network.py +++ b/autoPyTorch/pipeline/components/setup/network/base_network.py @@ -134,13 +134,15 @@ def _predict(self, network: torch.nn.Module, loader: torch.utils.data.DataLoader # Batch prediction Y_batch_preds = list() - for i, (X_batch, Y_batch) in enumerate(loader): - # Predict on batch - X_batch = X_batch.float().to(self.device) - Y_batch_pred = network(X_batch) - if self.final_activation is not None: - Y_batch_pred = self.final_activation(Y_batch_pred) - Y_batch_preds.append(Y_batch_pred.detach().cpu()) + # `torch.no_grad` reduces memory usage even after `model.eval()` + with torch.no_grad(): + for i, (X_batch, Y_batch) in enumerate(loader): + # Predict on batch + X_batch = X_batch.float().to(self.device) + Y_batch_pred = network(X_batch) + if self.final_activation is not None: + Y_batch_pred = self.final_activation(Y_batch_pred) + Y_batch_preds.append(Y_batch_pred.detach().cpu()) return torch.cat(Y_batch_preds, 0) diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py index 5388bfcc4..5f71825be 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py @@ -140,6 +140,14 @@ def get_hyperparameter_search_space( value_range=(True, False), default_value=True, ), + shake_shake_update_func: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="shake_shake_update_func", + value_range=('shake-shake', + 'shake-even', + 'even-even', + 'M3'), + default_value='shake-shake', + ), use_shake_drop: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_shake_drop", value_range=(True, False), default_value=True, @@ -181,16 +189,25 @@ def get_hyperparameter_search_space( if skip_connection_flag: + shake_shake_flag = 'shake-shake' in multi_branch_choice.value_range shake_drop_prob_flag = 'shake-drop' in multi_branch_choice.value_range mb_choice = get_hyperparameter(multi_branch_choice, CategoricalHyperparameter) cs.add_hyperparameter(mb_choice) cs.add_condition(CS.EqualsCondition(mb_choice, use_sc, True)) + shake_shake_update_func_conditional: List[str] = list() if shake_drop_prob_flag: shake_drop_prob = get_hyperparameter(max_shake_drop_probability, UniformFloatHyperparameter) cs.add_hyperparameter(shake_drop_prob) cs.add_condition(CS.EqualsCondition(shake_drop_prob, mb_choice, "shake-drop")) + shake_shake_update_func_conditional.append('shake-drop') + if shake_shake_flag: + shake_shake_update_func_conditional.append('shake-shake') + if len(shake_shake_update_func_conditional) > 0: + method = get_hyperparameter(shake_shake_update_func, CategoricalHyperparameter) + cs.add_hyperparameter(method) + cs.add_condition(CS.InCondition(method, mb_choice, shake_shake_update_func_conditional)) # It is the upper bound of the nr of groups, # since the configuration will actually be sampled. @@ -259,7 +276,7 @@ def __init__( # if in != out the shortcut needs a linear layer to match the result dimensions # if the shortcut needs a layer we apply batchnorm and activation to the shortcut # as well (start_norm) - if in_features != out_features: + if in_features != out_features and self.config["use_skip_connection"]: self.shortcut = nn.Linear(in_features, out_features) initial_normalization = list() if self.config['use_batch_norm']: @@ -289,13 +306,6 @@ def _build_block(self, in_features: int, out_features: int) -> nn.Module: if self.config['use_batch_norm']: layers.append(nn.BatchNorm1d(in_features)) layers.append(self.activation()) - elif not self.config['use_skip_connection']: - # if start norm is not None and skip connection is False - # we will never apply the start_norm for the first layer in the block, - # which is why we should account for this case. - if self.config['use_batch_norm']: - layers.append(nn.BatchNorm1d(in_features)) - layers.append(self.activation()) layers.append(nn.Linear(in_features, out_features)) @@ -311,9 +321,6 @@ def _build_block(self, in_features: int, out_features: int) -> nn.Module: def forward(self, x: torch.FloatTensor) -> torch.FloatTensor: - if self.config["use_skip_connection"]: - residual = x - # if shortcut is not none we need a layer such that x matches the output dimension if self.shortcut is not None and self.start_norm is not None: # in this case self.start_norm is also != none @@ -321,38 +328,42 @@ def forward(self, x: torch.FloatTensor) -> torch.FloatTensor: # in front of shortcut and layers. Note that in this case layers # does not start with batchnorm+activation but with the first linear layer # (see _build_block). As a result if in_features == out_features - # -> result = x + W(~D(A(BN(W(A(BN(x)))))) + # -> result = x + W_2(~D(A(BN(W_1(A(BN(x)))))) # if in_features != out_features # -> result = W_shortcut(A(BN(x))) + W_2(~D(A(BN(W_1(A(BN(x)))))) x = self.start_norm(x) - if self.config["use_skip_connection"]: - residual = self.shortcut(x) - - # TODO make the below code better - if self.config["use_skip_connection"]: - if self.config["multi_branch_choice"] == 'shake-shake': - x1 = self.layers(x) - x2 = self.shake_shake_layers(x) - alpha, beta = shake_get_alpha_beta(self.training, x.is_cuda) - x = shake_shake(x1, x2, alpha, beta) - else: - x = self.layers(x) + residual = self.shortcut(x) + elif self.config["use_skip_connection"]: + # We use a skip connection but we do not need to match dimensions + residual = x + else: # Early-return because no need of skip connection + return self.layers(x) + + if self.config["multi_branch_choice"] == 'shake-shake': + x1 = self.layers(x) + x2 = self.shake_shake_layers(x) + alpha, beta = shake_get_alpha_beta( + is_training=self.training, + is_cuda=x.is_cuda, + method=self.config['shake_shake_update_func'], + ) + x = shake_shake(x1, x2, alpha, beta) + elif self.config["multi_branch_choice"] == 'shake-drop': + x = self.layers(x) + alpha, beta = shake_get_alpha_beta( + is_training=self.training, + is_cuda=x.is_cuda, + method=self.config['shake_shake_update_func'], + ) + bl = shake_drop_get_bl( + self.block_index, + 1 - self.config["max_shake_drop_probability"], + self.num_blocks, + self.training, + x.is_cuda, + ) + x = shake_drop(x, alpha, beta, bl) else: x = self.layers(x) - if self.config["use_skip_connection"]: - if self.config["multi_branch_choice"] == 'shake-drop': - alpha, beta = shake_get_alpha_beta(self.training, x.is_cuda) - bl = shake_drop_get_bl( - self.block_index, - 1 - self.config["max_shake_drop_probability"], - self.num_blocks, - self.training, - x.is_cuda, - ) - x = shake_drop(x, alpha, beta, bl) - - if self.config["use_skip_connection"]: - x = x + residual - - return x + return x + residual diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py index a9e1f011e..2e4fa53c5 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py @@ -73,6 +73,7 @@ def build_backbone(self, input_shape: Tuple[int, ...]) -> torch.nn.Sequential: ) if self.config['use_batch_norm']: layers.append(torch.nn.BatchNorm1d(self.config["num_units_%i" % self.config['num_groups']])) + layers.append(_activations[self.config["activation"]]()) backbone = torch.nn.Sequential(*layers) return backbone @@ -145,6 +146,14 @@ def get_hyperparameter_search_space( # type: ignore[override] 'stairs'), default_value='funnel', ), + shake_shake_update_func: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="shake_shake_update_func", + value_range=('shake-shake', + 'shake-even', + 'even-even', + 'M3'), + default_value='shake-shake', + ), max_shake_drop_probability: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="max_shake_drop_probability", value_range=(0, 1), @@ -188,17 +197,24 @@ def get_hyperparameter_search_space( # type: ignore[override] if skip_connection_flag: - shake_drop_prob_flag = False - if 'shake-drop' in multi_branch_choice.value_range: - shake_drop_prob_flag = True + shake_shake_flag = 'shake-shake' in multi_branch_choice.value_range + shake_drop_prob_flag = 'shake-drop' in multi_branch_choice.value_range mb_choice = get_hyperparameter(multi_branch_choice, CategoricalHyperparameter) cs.add_hyperparameter(mb_choice) cs.add_condition(CS.EqualsCondition(mb_choice, use_sc, True)) + shake_shake_update_func_conditional: List[str] = list() if shake_drop_prob_flag: shake_drop_prob = get_hyperparameter(max_shake_drop_probability, UniformFloatHyperparameter) cs.add_hyperparameter(shake_drop_prob) cs.add_condition(CS.EqualsCondition(shake_drop_prob, mb_choice, "shake-drop")) + shake_shake_update_func_conditional.append('shake-drop') + if shake_shake_flag: + shake_shake_update_func_conditional.append('shake-shake') + if len(shake_shake_update_func_conditional) > 0: + method = get_hyperparameter(shake_shake_update_func, CategoricalHyperparameter) + cs.add_hyperparameter(method) + cs.add_condition(CS.InCondition(method, mb_choice, shake_shake_update_func_conditional)) return cs diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/utils.py b/autoPyTorch/pipeline/components/setup/network_backbone/utils.py index 0539df422..80f8e4dc0 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/utils.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/utils.py @@ -29,10 +29,15 @@ def get_output_shape(network: torch.nn.Module, input_shape: Tuple[int, ...], has """ placeholder = torch.randn((2, *input_shape), dtype=torch.float) with torch.no_grad(): +<<<<<<< HEAD if has_hidden_states: output = network(placeholder)[0] else: output = network(placeholder) +======= + output = network(placeholder) + +>>>>>>> Bug fixes (#249) return tuple(output.shape[1:]) @@ -87,7 +92,11 @@ class ShakeDropFunction(Function): Github URL: https://github.com/owruby/shake-drop_pytorch/blob/master/models/shakedrop.py """ @staticmethod +<<<<<<< HEAD def forward(ctx: Any, +======= + def forward(ctx: typing.Any, +>>>>>>> Bug fixes (#249) x: torch.Tensor, alpha: torch.Tensor, beta: torch.Tensor, @@ -114,15 +123,31 @@ def backward(ctx: Any, shake_drop = ShakeDropFunction.apply +<<<<<<< HEAD def shake_get_alpha_beta(is_training: bool, is_cuda: bool ) -> Tuple[torch.Tensor, torch.Tensor]: """ The methods used in this function have been introduced in 'ShakeShake Regularisation' Currently, this function supports `shake-shake`. +======= +def shake_get_alpha_beta( + is_training: bool, + is_cuda: bool, + method: str +) -> typing.Tuple[torch.Tensor, torch.Tensor]: + """ + The methods used in this function have been introduced in 'ShakeShake Regularisation' + Each method name is available in the referred paper. + Currently, this function supports `even-even`, `shake-even`, `shake-shake` and `M3`. +>>>>>>> Bug fixes (#249) Args: is_training (bool): Whether the computation for the training is_cuda (bool): Whether the tensor is on CUDA +<<<<<<< HEAD +======= + method (str): The shake method either `even-even`, `shake-even`, `shake-shake` or `M3` +>>>>>>> Bug fixes (#249) Returns: alpha, beta (Tuple[float, float]): @@ -134,17 +159,34 @@ def shake_get_alpha_beta(is_training: bool, is_cuda: bool Author: Xavier Gastaldi URL: https://arxiv.org/abs/1705.07485 +<<<<<<< HEAD Note: The names have been taken from the paper as well. Currently, this function supports `shake-shake`. +======= + The names have been taken from the paper as well. + Currently, this function supports `even-even`, `shake-even`, `shake-shake` and `M3`. +>>>>>>> Bug fixes (#249) """ if not is_training: result = (torch.FloatTensor([0.5]), torch.FloatTensor([0.5])) return result if not is_cuda else (result[0].cuda(), result[1].cuda()) # TODO implement other update methods - alpha = torch.rand(1) - beta = torch.rand(1) + # alpha is the weight ratio for the forward pass and beta is that for the backward pass + alpha = torch.FloatTensor([0.5]) if method.startswith('even') else torch.rand(1) + if method.endswith('even'): + beta = torch.FloatTensor([0.5]) + elif method.endswith('shake'): + beta = torch.rand(1) + elif method == 'M3': + # Table 4 in the paper `Shake-Shake regularization` + rnd = torch.rand(1) + beta = torch.FloatTensor( + [rnd * (0.5 - alpha) + alpha if alpha < 0.5 else rnd * (alpha - 0.5) + 0.5] + ) + else: + raise ValueError(f"Unknown method `{method}` for ShakeShakeRegularisation in NetworkBackbone") if is_cuda: alpha = alpha.cuda() @@ -154,16 +196,27 @@ def shake_get_alpha_beta(is_training: bool, is_cuda: bool def shake_drop_get_bl( +<<<<<<< HEAD block_index: int, min_prob_no_shake: float, num_blocks: int, is_training: bool, is_cuda: bool +======= + block_index: int, + min_prob_no_shake: float, + num_blocks: int, + is_training: bool, + is_cuda: bool +>>>>>>> Bug fixes (#249) ) -> torch.Tensor: """ The sampling of Bernoulli random variable based on Eq. (4) in the paper +<<<<<<< HEAD +======= +>>>>>>> Bug fixes (#249) Args: block_index (int): The index of the block from the input layer min_prob_no_shake (float): The initial shake probability @@ -173,18 +226,28 @@ def shake_drop_get_bl( Returns: bl (torch.Tensor): a Bernoulli random variable in {0, 1} +<<<<<<< HEAD +======= +>>>>>>> Bug fixes (#249) Reference: ShakeDrop Regularization for Deep Residual Learning Yoshihiro Yamada et. al. (2020) paper: https://arxiv.org/pdf/1802.02375.pdf implementation: https://github.com/imenurok/ShakeDrop """ +<<<<<<< HEAD pl = 1 - ((block_index + 1) / num_blocks) * (1 - min_prob_no_shake) if is_training: # Move to torch.rand(1) for reproducibility +======= + pl = 1 - ((block_index + 1) / num_blocks) * (1 - min_prob_no_shake) + + if is_training: + # Move to torch.randn(1) for reproducibility +>>>>>>> Bug fixes (#249) bl = torch.as_tensor(1.0) if torch.rand(1) <= pl else torch.as_tensor(0.0) else: bl = torch.as_tensor(pl) diff --git a/autoPyTorch/pipeline/components/setup/network_embedding/base_network_embedding.py b/autoPyTorch/pipeline/components/setup/network_embedding/base_network_embedding.py index e113d5774..4261cad84 100644 --- a/autoPyTorch/pipeline/components/setup/network_embedding/base_network_embedding.py +++ b/autoPyTorch/pipeline/components/setup/network_embedding/base_network_embedding.py @@ -1,5 +1,10 @@ +<<<<<<< HEAD import copy from typing import Any, Dict, List, Optional, Tuple, Union +======= +# import copy +from typing import Any, Dict, Optional, Tuple +>>>>>>> Bug fixes (#249) import numpy as np @@ -48,7 +53,7 @@ def build_embedding(self, num_numerical_features: int) -> Tuple[nn.Module, Optional[List[int]]]: raise NotImplementedError - def _get_args(self, X: Dict[str, Any]) -> Tuple[int, np.ndarray]: + def _get_args(self, X: Dict[str, Any]) -> Tuple[None, None]: # Tuple[int, np.ndarray]: # Feature preprocessors can alter numerical columns if len(X['dataset_properties']['numerical_columns']) == 0: num_numerical_columns = 0 diff --git a/autoPyTorch/pipeline/components/setup/network_head/no_head.py b/autoPyTorch/pipeline/components/setup/network_head/no_head.py index 870f680fb..0e711f06c 100644 --- a/autoPyTorch/pipeline/components/setup/network_head/no_head.py +++ b/autoPyTorch/pipeline/components/setup/network_head/no_head.py @@ -23,7 +23,6 @@ def build_head(self, input_shape: Tuple[int, ...], output_shape: Tuple[int, ...] layers = [] in_features = np.prod(input_shape).item() out_features = np.prod(output_shape).item() - layers.append(_activations[self.config["activation"]]()) layers.append(nn.Linear(in_features=in_features, out_features=out_features)) return nn.Sequential(*layers) diff --git a/autoPyTorch/pipeline/components/setup/optimizer/AdamWOptimizer.py b/autoPyTorch/pipeline/components/setup/optimizer/AdamWOptimizer.py index 3ae84a9e0..348fb4925 100644 --- a/autoPyTorch/pipeline/components/setup/optimizer/AdamWOptimizer.py +++ b/autoPyTorch/pipeline/components/setup/optimizer/AdamWOptimizer.py @@ -97,9 +97,9 @@ def get_hyperparameter_search_space( default_value=True, ), weight_decay: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="weight_decay", - value_range=(1E-7, 0.1), + value_range=(1E-5, 0.1), default_value=1E-4, - log=True), + log=False), ) -> ConfigurationSpace: cs = ConfigurationSpace() diff --git a/autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py b/autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py index a8651e158..c601e4a3d 100644 --- a/autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py +++ b/autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py @@ -115,7 +115,7 @@ def fit(self, X: Dict[str, Any], y: Any = None) -> torch.utils.data.DataLoader: shuffle=True, num_workers=X.get('num_workers', 0), pin_memory=X.get('pin_memory', True), - drop_last=X.get('drop_last', True), + drop_last=X.get('drop_last', False), collate_fn=custom_collate_fn, ) @@ -149,6 +149,8 @@ def get_loader(self, X: np.ndarray, y: Optional[np.ndarray] = None, batch_size: train_tensors=(X, y), seed=self.random_state.get_state()[1][0], # This dataset is used for loading test data in a batched format + seed=self.random_state.get_state()[1][0], + shuffle=False, train_transforms=self.test_transform, val_transforms=self.test_transform, ) diff --git a/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py b/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py index 157924ed0..7f5385382 100644 --- a/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py @@ -31,7 +31,7 @@ class AdversarialTrainer(BaseTrainerComponent): def __init__( self, epsilon: float, - weighted_loss: bool = False, + weighted_loss: int = 0, random_state: Optional[np.random.RandomState] = None, use_stochastic_weight_averaging: bool = False, use_snapshot_ensemble: bool = False, @@ -157,7 +157,7 @@ def get_properties(dataset_properties: Optional[Dict[str, Any]] = None 'shortname': 'AdversarialTrainer', 'name': 'AdversarialTrainer', 'handles_tabular': True, - 'handles_image': False, + 'handles_image': True, 'handles_time_series': False, } @@ -166,8 +166,8 @@ def get_hyperparameter_search_space( dataset_properties: Optional[Dict] = None, weighted_loss: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="weighted_loss", - value_range=(True, False), - default_value=True), + value_range=(1, ), + default_value=1), la_steps: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="la_steps", value_range=(5, 10), @@ -192,16 +192,21 @@ def get_hyperparameter_search_space( default_value=True), se_lastk: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="se_lastk", - value_range=(3,), + value_range=(3, ), default_value=3), epsilon: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="epsilon", - value_range=(0.05, 0.2), - default_value=0.2), + value_range=(0.001, 0.15), + default_value=0.007, + log=True), ) -> ConfigurationSpace: cs = ConfigurationSpace() + epsilon = HyperparameterSearchSpace(hyperparameter="epsilon", + value_range=(0.007, 0.007), + default_value=0.007) add_hyperparameter(cs, epsilon, UniformFloatHyperparameter) + add_hyperparameter(cs, use_stochastic_weight_averaging, CategoricalHyperparameter) snapshot_ensemble_flag = any(use_snapshot_ensemble.value_range) @@ -229,9 +234,17 @@ def get_hyperparameter_search_space( parent_hyperparameter=parent_hyperparameter ) + """ # TODO, decouple the weighted loss from the trainer if dataset_properties is not None: if STRING_TO_TASK_TYPES[dataset_properties['task_type']] in CLASSIFICATION_TASKS: add_hyperparameter(cs, weighted_loss, CategoricalHyperparameter) + """ + # TODO, decouple the weighted loss from the trainer. Uncomment the code above and + # remove the code below. Also update the method signature, so the weighted loss + # is not a constant. + if dataset_properties is not None: + if STRING_TO_TASK_TYPES[dataset_properties['task_type']] in CLASSIFICATION_TASKS: + add_hyperparameter(cs, weighted_loss, Constant) return cs diff --git a/autoPyTorch/pipeline/components/training/trainer/RowCutMixTrainer.py b/autoPyTorch/pipeline/components/training/trainer/RowCutMixTrainer.py index e36faf121..3f7866f3c 100644 --- a/autoPyTorch/pipeline/components/training/trainer/RowCutMixTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/RowCutMixTrainer.py @@ -1,4 +1,4 @@ -import typing +from typing import Any, Dict, Optional, Tuple, Union import numpy as np @@ -11,7 +11,7 @@ class RowCutMixTrainer(MixUp, BaseTrainerComponent): def data_preparation(self, X: np.ndarray, y: np.ndarray, - ) -> typing.Tuple[np.ndarray, typing.Dict[str, np.ndarray]]: + ) -> Tuple[np.ndarray, Dict[str, np.ndarray]]: """ Depending on the trainer choice, data fed to the network might be pre-processed on a different way. That is, in standard training we provide the data to the @@ -26,39 +26,38 @@ def data_preparation(self, X: np.ndarray, y: np.ndarray, np.ndarray: that processes data typing.Dict[str, np.ndarray]: arguments to the criterion function """ - alpha, beta = 1.0, 1.0 - lam = self.random_state.beta(alpha, beta) - batch_size = X.shape[0] - device = torch.device('cuda' if X.is_cuda else 'cpu') - permed_indices = torch.randperm(batch_size).to(device) + beta = 1.0 + lam = self.random_state.beta(beta, beta) + batch_size, n_columns = np.shape(X) + # shuffled_indices: Shuffled version of torch.arange(batch_size) + shuffled_indices = torch.randperm(batch_size).cuda() if X.is_cuda else torch.randperm(batch_size) r = self.random_state.rand(1) if beta <= 0 or r > self.alpha: - return X, {'y_a': y, 'y_b': y[permed_indices], 'lam': 1} + return X, {'y_a': y, 'y_b': y[shuffled_indices], 'lam': 1} - # batch_size (permutation of rows), col_size = X.shape - col_size = X.shape[1] - col_indices = torch.tensor( + cut_column_indices = torch.as_tensor( self.random_state.choice( - range(col_size), - max(1, int(col_size * lam)), - replace=False - ) + range(n_columns), + max(1, np.int32(n_columns * lam)), + replace=False, + ), ) - # Replace selected columns with columns from another data point - X[:, col_indices] = X[permed_indices, :][:, col_indices] + # Replace the values in `cut_indices` columns with + # the values from `permed_indices` + X[:, cut_column_indices] = X[shuffled_indices, :][:, cut_column_indices] - # Adjust lam - lam = 1 - len(col_indices) / X.shape[1] + # Since we cannot cut exactly `lam x 100 %` of rows, we need to adjust the `lam` + lam = 1 - (len(cut_column_indices) / n_columns) - y_a, y_b = y, y[permed_indices] + y_a, y_b = y, y[shuffled_indices] return X, {'y_a': y_a, 'y_b': y_b, 'lam': lam} @staticmethod - def get_properties(dataset_properties: typing.Optional[typing.Dict[str, typing.Any]] = None - ) -> typing.Dict[str, typing.Union[str, bool]]: + def get_properties(dataset_properties: Optional[Dict[str, Any]] = None + ) -> Dict[str, Union[str, bool]]: return { 'shortname': 'RowCutMixTrainer', 'name': 'MixUp Regularized with Cutoff Tabular Trainer', diff --git a/autoPyTorch/pipeline/components/training/trainer/RowCutOutTrainer.py b/autoPyTorch/pipeline/components/training/trainer/RowCutOutTrainer.py index 5e1b9a1a3..4578082cb 100644 --- a/autoPyTorch/pipeline/components/training/trainer/RowCutOutTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/RowCutOutTrainer.py @@ -1,9 +1,7 @@ -import typing +from typing import Any, Dict, Optional, Tuple, Union import numpy as np -import torch - from autoPyTorch.pipeline.components.training.trainer.base_trainer import BaseTrainerComponent from autoPyTorch.pipeline.components.training.trainer.cutout_utils import CutOut @@ -17,13 +15,8 @@ class RowCutOutTrainer(CutOut, BaseTrainerComponent): Github URL: https://github.com/hysts/pytorch_cutout/blob/master/dataloader.py#L36-L68 """ - # 0 is non-informative in image data - NUMERICAL_VALUE = 0 - # -1 is the conceptually equivalent to 0 in a image, i.e. 0-pad - CATEGORICAL_VALUE = -1 - def data_preparation(self, X: np.ndarray, y: np.ndarray, - ) -> typing.Tuple[np.ndarray, typing.Dict[str, np.ndarray]]: + ) -> Tuple[np.ndarray, Dict[str, np.ndarray]]: """ Depending on the trainer choice, data fed to the network might be pre-processed on a different way. That is, in standard training we provide the data to the @@ -36,9 +29,8 @@ def data_preparation(self, X: np.ndarray, y: np.ndarray, Returns: np.ndarray: that processes data - typing.Dict[str, np.ndarray]: arguments to the criterion function + Dict[str, np.ndarray]: arguments to the criterion function """ - r = self.random_state.rand(1) if r > self.cutout_prob: y_a = y @@ -46,30 +38,23 @@ def data_preparation(self, X: np.ndarray, y: np.ndarray, lam = 1 return X, {'y_a': y_a, 'y_b': y_b, 'lam': lam} - # (batch_size (permutation of rows), col_size) = X.shape - col_size = X.shape[1] - col_indices = self.random_state.choice(range(col_size), max(1, int(col_size * self.patch_ratio)), - replace=False) - - if not isinstance(self.numerical_columns, typing.Iterable): - raise ValueError("numerical_columns in {} must be iterable, " - "but got {}.".format(self.__class__.__name__, - self.numerical_columns)) - - numerical_indices = torch.tensor(self.numerical_columns) - categorical_indices = torch.tensor([idx for idx in col_indices if idx not in self.numerical_columns]) - - X[:, categorical_indices.long()] = self.CATEGORICAL_VALUE - X[:, numerical_indices.long()] = self.NUMERICAL_VALUE + size: int = np.shape(X)[1] + cut_column_indices = self.random_state.choice( + range(size), + max(1, np.int32(size * self.patch_ratio)), + replace=False, + ) + # Mask the selected features as 0 + X[:, cut_column_indices] = 0 lam = 1 y_a = y y_b = y return X, {'y_a': y_a, 'y_b': y_b, 'lam': lam} @staticmethod - def get_properties(dataset_properties: typing.Optional[typing.Dict[str, typing.Any]] = None - ) -> typing.Dict[str, typing.Union[str, bool]]: + def get_properties(dataset_properties: Optional[Dict[str, Any]] = None + ) -> Dict[str, Union[str, bool]]: return { 'shortname': 'RowCutOutTrainer', 'name': 'RowCutOutTrainer', diff --git a/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py b/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py index 9e44399fd..fc5cc3e3a 100644 --- a/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py @@ -14,7 +14,7 @@ class StandardTrainer(BaseTrainerComponent): def __init__(self, - weighted_loss: bool = False, + weighted_loss: int = 0, use_stochastic_weight_averaging: bool = False, use_snapshot_ensemble: bool = False, se_lastk: int = 3, @@ -25,7 +25,7 @@ def __init__(self, This class handles the training of a network for a single given epoch. Args: - weighted_loss (bool): whether to use weighted loss + weighted_loss (int): whether to use weighted loss """ super().__init__(random_state=random_state, diff --git a/autoPyTorch/pipeline/components/training/trainer/__init__.py b/autoPyTorch/pipeline/components/training/trainer/__init__.py index 1ff296855..282d356b8 100755 --- a/autoPyTorch/pipeline/components/training/trainer/__init__.py +++ b/autoPyTorch/pipeline/components/training/trainer/__init__.py @@ -287,7 +287,6 @@ def fit(self, X: Dict[str, Any], y: Any = None, **kwargs: Any) -> autoPyTorchCom self.choice: autoPyTorchComponent = cast(autoPyTorchComponent, self.choice) if self.choice.use_snapshot_ensemble: X['network_snapshots'].extend(self.choice.model_snapshots) - return self.choice def prepare_trainer(self, X: Dict) -> None: @@ -442,14 +441,15 @@ def _fit(self, X: Dict[str, Any], y: Any = None, **kwargs: Any) -> 'TrainerChoic raise RuntimeError("Budget exhausted without finishing an epoch.") if self.choice.use_stochastic_weight_averaging and self.choice.swa_updated: + # update batch norm statistics swa_utils.update_bn(loader=X['train_data_loader'], model=self.choice.swa_model.double()) # change model update_model_state_dict_from_swa(X['network'], self.choice.swa_model.state_dict()) if self.choice.use_snapshot_ensemble: - for model in self.choice.model_snapshots: - swa_utils.update_bn(loader=X['train_data_loader'], model=model.double()) + # we update only the last network which pertains to the stochastic weight averaging model + swa_utils.update_bn(X['train_data_loader'], self.choice.model_snapshots[-1].double()) # wrap up -- add score if not evaluating every epoch if not self.eval_valid_each_epoch(X): diff --git a/autoPyTorch/pipeline/components/training/trainer/base_trainer.py b/autoPyTorch/pipeline/components/training/trainer/base_trainer.py index 9e39b63cd..07c3877e2 100644 --- a/autoPyTorch/pipeline/components/training/trainer/base_trainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/base_trainer.py @@ -213,7 +213,7 @@ class BaseTrainerComponent(autoPyTorchTrainingComponent): """ Base class for training Args: - weighted_loss (bool, default=False): In case for classification, whether to weight + weighted_loss (int, default=0): In case for classification, whether to weight the loss function according to the distribution of classes in the target use_stochastic_weight_averaging (bool, default=True): whether to use stochastic weight averaging. Stochastic weight averaging is a simple average of @@ -228,7 +228,7 @@ class BaseTrainerComponent(autoPyTorchTrainingComponent): random_state: **lookahead_config: """ - def __init__(self, weighted_loss: bool = False, + def __init__(self, weighted_loss: int = 0, use_stochastic_weight_averaging: bool = True, use_snapshot_ensemble: bool = True, se_lastk: int = 3, @@ -360,8 +360,12 @@ def on_epoch_end(self, X: Dict[str, Any], epoch: int) -> bool: if self.use_snapshot_ensemble: assert self.model_snapshots is not None, "model snapshots container can't be " \ "none when snapshot ensembling is enabled" - model_copy = deepcopy(self.swa_model) if self.use_stochastic_weight_averaging \ - else deepcopy(self.model) + is_last_epoch = (epoch == self.budget_tracker.max_epochs) + if is_last_epoch and self.use_stochastic_weight_averaging: + model_copy = deepcopy(self.swa_model) + else: + model_copy = deepcopy(self.model) + assert model_copy is not None model_copy.cpu() self.model_snapshots.append(model_copy) @@ -601,8 +605,8 @@ def get_hyperparameter_search_space( dataset_properties: Optional[Dict] = None, weighted_loss: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="weighted_loss", - value_range=(True, False), - default_value=True), + value_range=(1, ), + default_value=1), la_steps: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="la_steps", value_range=(5, 10), @@ -627,7 +631,7 @@ def get_hyperparameter_search_space( default_value=True), se_lastk: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="se_lastk", - value_range=(3,), + value_range=(3, ), default_value=3), ) -> ConfigurationSpace: cs = ConfigurationSpace() @@ -658,9 +662,17 @@ def get_hyperparameter_search_space( parent_hyperparameter=parent_hyperparameter ) + """ # TODO, decouple the weighted loss from the trainer if dataset_properties is not None: if STRING_TO_TASK_TYPES[dataset_properties['task_type']] in CLASSIFICATION_TASKS: add_hyperparameter(cs, weighted_loss, CategoricalHyperparameter) + """ + # TODO, decouple the weighted loss from the trainer. Uncomment the code above and + # remove the code below. Also update the method signature, so the weighted loss + # is not a constant. + if dataset_properties is not None: + if STRING_TO_TASK_TYPES[dataset_properties['task_type']] in CLASSIFICATION_TASKS: + add_hyperparameter(cs, weighted_loss, Constant) return cs diff --git a/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py b/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py index 1b987d599..4feedf5cb 100644 --- a/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py +++ b/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py @@ -20,7 +20,7 @@ class CutOut: def __init__(self, patch_ratio: float, cutout_prob: float, - weighted_loss: bool = False, + weighted_loss: int = 0, random_state: Optional[np.random.RandomState] = None, use_stochastic_weight_averaging: bool = False, use_snapshot_ensemble: bool = False, @@ -63,8 +63,8 @@ def get_hyperparameter_search_space( dataset_properties: Optional[Dict] = None, weighted_loss: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="weighted_loss", - value_range=(True, False), - default_value=True), + value_range=(1, ), + default_value=1), la_steps: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="la_steps", value_range=(5, 10), @@ -136,9 +136,17 @@ def get_hyperparameter_search_space( parent_hyperparameter=parent_hyperparameter ) + """ # TODO, decouple the weighted loss from the trainer if dataset_properties is not None: if STRING_TO_TASK_TYPES[dataset_properties['task_type']] in CLASSIFICATION_TASKS: add_hyperparameter(cs, weighted_loss, CategoricalHyperparameter) + """ + # TODO, decouple the weighted loss from the trainer. Uncomment the code above and + # remove the code below. Also update the method signature, so the weighted loss + # is not a constant. + if dataset_properties is not None: + if STRING_TO_TASK_TYPES[dataset_properties['task_type']] in CLASSIFICATION_TASKS: + add_hyperparameter(cs, weighted_loss, Constant) return cs diff --git a/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py b/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py index e33011bf5..e2ea25148 100644 --- a/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py +++ b/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py @@ -26,7 +26,7 @@ class MixUp: Github URL: https://github.com/facebookresearch/mixup-cifar10/blob/master/train.py#L119-L138 """ def __init__(self, alpha: float, - weighted_loss: bool = False, + weighted_loss: int = 0, random_state: Optional[np.random.RandomState] = None, use_stochastic_weight_averaging: bool = False, use_snapshot_ensemble: bool = False, @@ -68,8 +68,8 @@ def get_hyperparameter_search_space( dataset_properties: Optional[Dict] = None, weighted_loss: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="weighted_loss", - value_range=(True, False), - default_value=True), + value_range=(1, ), + default_value=1), la_steps: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="la_steps", value_range=(5, 10), @@ -94,7 +94,7 @@ def get_hyperparameter_search_space( default_value=True), se_lastk: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="se_lastk", - value_range=(3,), + value_range=(3, ), default_value=3), alpha: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="alpha", @@ -134,9 +134,18 @@ def get_hyperparameter_search_space( la_config_space, parent_hyperparameter=parent_hyperparameter ) + + """ # TODO, decouple the weighted loss from the trainer if dataset_properties is not None: if STRING_TO_TASK_TYPES[dataset_properties['task_type']] in CLASSIFICATION_TASKS: add_hyperparameter(cs, weighted_loss, CategoricalHyperparameter) + """ + # TODO, decouple the weighted loss from the trainer. Uncomment the code above and + # remove the code below. Also update the method signature, so the weighted loss + # is not a constant. + if dataset_properties is not None: + if STRING_TO_TASK_TYPES[dataset_properties['task_type']] in CLASSIFICATION_TASKS: + add_hyperparameter(cs, weighted_loss, Constant) return cs diff --git a/autoPyTorch/pipeline/tabular_classification.py b/autoPyTorch/pipeline/tabular_classification.py index 720d0af64..bc9616f58 100644 --- a/autoPyTorch/pipeline/tabular_classification.py +++ b/autoPyTorch/pipeline/tabular_classification.py @@ -142,10 +142,23 @@ def _predict_proba(self, X: np.ndarray) -> np.ndarray: # The final layer is always softmax now (`pred` already gives pseudo proba) return pred else: +<<<<<<< HEAD raise ValueError("Expected output_shape to be integer, got {}," "Tabular Classification only supports 'binary' and 'multiclass' outputs" "got {}".format(type(self.dataset_properties['output_shape']), self.dataset_properties['output_type'])) +======= + all_proba = [] + + for k in range(self.dataset_properties['output_shape']): + proba_k = pred[:, k, :self.dataset_properties['output_shape'][k]] + normalizer = proba_k.sum(axis=1)[:, np.newaxis] + normalizer[normalizer == 0.0] = 1.0 + proba_k /= normalizer + all_proba.append(proba_k) + + return np.array(all_proba) +>>>>>>> Bug fixes (#249) def predict_proba(self, X: np.ndarray, batch_size: Optional[int] = None) -> np.ndarray: """predict_proba. diff --git a/autoPyTorch/utils/backend.py b/autoPyTorch/utils/backend.py new file mode 100644 index 000000000..5348bd11c --- /dev/null +++ b/autoPyTorch/utils/backend.py @@ -0,0 +1,575 @@ +import glob +import os +import pickle +import re +import shutil +import tempfile +import time +import uuid +import warnings +from typing import Dict, List, Optional, Tuple, Union + +import lockfile + +import numpy as np + +from autoPyTorch.datasets.base_dataset import BaseDataset +from autoPyTorch.ensemble.abstract_ensemble import AbstractEnsemble +from autoPyTorch.pipeline.base_pipeline import BasePipeline +from autoPyTorch.utils.logging_ import PicklableClientLogger, get_named_client_logger + +__all__ = [ + 'Backend' +] + + +def create( + temporary_directory: Optional[str], + output_directory: Optional[str], + delete_tmp_folder_after_terminate: bool = True, + delete_output_folder_after_terminate: bool = True, +) -> 'Backend': + """ + Creates a backend object that manages disk related transactions + + Args: + temporary_directory (str): where all temporal data is to be dumped + output_directory (str): where all predictions are to be output + delete_tmp_folder_after_terminate (bool): whether to delete the + temporal directory when then run completes + delete_output_folder_after_terminate (bool): whether to delete + the output directory when the run completes + + Returns: + Backend object + """ + context = BackendContext(temporary_directory, output_directory, + delete_tmp_folder_after_terminate, + delete_output_folder_after_terminate, + ) + backend = Backend(context) + + return backend + + +def get_randomized_directory_name(temporary_directory: Optional[str] = None) -> str: + uuid_str = str(uuid.uuid1(clock_seq=os.getpid())) + + temporary_directory = ( + temporary_directory + if temporary_directory + else os.path.join( + tempfile.gettempdir(), + "autoPyTorch_tmp_{}".format( + uuid_str, + ), + ) + ) + + return temporary_directory + + +class BackendContext(object): + + def __init__(self, + temporary_directory: Optional[str], + output_directory: Optional[str], + delete_tmp_folder_after_terminate: bool, + delete_output_folder_after_terminate: bool, + ): + + # Check that the names of tmp_dir and output_dir is not the same. + if temporary_directory == output_directory and temporary_directory is not None: + raise ValueError("The temporary and the output directory " + "must be different.") + + self.delete_tmp_folder_after_terminate = delete_tmp_folder_after_terminate + self.delete_output_folder_after_terminate = delete_output_folder_after_terminate + # attributes to check that directories were created by autoPyTorch + self._tmp_dir_created = False + self._output_dir_created = False + + self._temporary_directory = ( + get_randomized_directory_name( + temporary_directory=temporary_directory, + ) + ) + self._output_directory = output_directory + self.create_directories() + self._logger = None # type: Optional[PicklableClientLogger] + + @property + def output_directory(self) -> Optional[str]: + if self._output_directory is not None: + # make sure that tilde does not appear on the path. + return os.path.expanduser(os.path.expandvars(self._output_directory)) + else: + return None + + @property + def temporary_directory(self) -> str: + # make sure that tilde does not appear on the path. + return os.path.expanduser(os.path.expandvars(self._temporary_directory)) + + def create_directories(self) -> None: + # Exception is raised if self.temporary_directory already exists. + os.makedirs(self.temporary_directory) + self._tmp_dir_created = True + + # Exception is raised if self.output_directory already exists. + if self.output_directory is not None: + os.makedirs(self.output_directory) + self._output_dir_created = True + + def delete_directories(self, force: bool = True) -> None: + if self.output_directory and (self.delete_output_folder_after_terminate or force): + if self._output_dir_created is False: + raise ValueError("Failed to delete output dir: %s because autoPyTorch did not " + "create it. Please make sure that the specified output dir does " + "not exist when instantiating autoPyTorch." + % self.output_directory) + try: + shutil.rmtree(self.output_directory) + except Exception: + try: + if self._logger is not None: + self._logger.warning("Could not delete output dir: %s" % + self.output_directory) + else: + warnings.warn("Could not delete output dir: %s" % self.output_directory) + except Exception: + warnings.warn("Could not delete output dir: %s" % self.output_directory) + + if self.delete_tmp_folder_after_terminate or force: + if self._tmp_dir_created is False: + raise ValueError("Failed to delete tmp dir: % s because autoPyTorch did not " + "create it. Please make sure that the specified tmp dir does not " + "exist when instantiating autoPyTorch." + % self.temporary_directory) + try: + shutil.rmtree(self.temporary_directory) + except Exception: + try: + if self._logger is not None: + self._logger.warning( + "Could not delete tmp dir: %s" % self.temporary_directory) + else: + warnings.warn("Could not delete tmp dir: %s" % self.temporary_directory) + except Exception: + warnings.warn("Could not delete tmp dir: %s" % self.temporary_directory) + + +class Backend(object): + """Utility class to load and save all objects to be persisted. + These are: + * start time of auto-pytorch + * true targets of the ensemble + """ + + def __init__(self, context: BackendContext): + self._logger = None # type: Optional[PicklableClientLogger] + self.context = context + + # Track the number of configurations launched + # num_run == 1 means a dummy estimator run + self.active_num_run = 1 + + # Create the temporary directory if it does not yet exist + try: + os.makedirs(self.temporary_directory) + except Exception: + pass + # This does not have to exist or be specified + if self.output_directory is not None: + if not os.path.exists(self.output_directory): + raise ValueError("Output directory %s does not exist." % self.output_directory) + + self.internals_directory = os.path.join(self.temporary_directory, ".autoPyTorch") + self._make_internals_directory() + + def setup_logger(self, name: str, port: int) -> None: + self._logger = get_named_client_logger( + name=name, + port=port, + ) + self.context._logger = self._logger + return + + @property + def output_directory(self) -> Optional[str]: + return self.context.output_directory + + @property + def temporary_directory(self) -> str: + return self.context.temporary_directory + + def _make_internals_directory(self) -> None: + # TODO: make exist_ok a function argument + try: + os.makedirs(self.internals_directory, exist_ok=True) + except Exception as e: + if self._logger is not None: + self._logger.debug("_make_internals_directory: %s" % e) + try: + os.makedirs(self.get_runs_directory(), exist_ok=True) + except Exception as e: + if self._logger is not None: + self._logger.debug("_make_internals_directory: %s" % e) + + def _get_start_time_filename(self, seed: Union[str, int]) -> str: + if isinstance(seed, str): + seed = int(seed) + return os.path.join(self.internals_directory, "start_time_%d" % seed) + + def save_start_time(self, seed: str) -> str: + self._make_internals_directory() + start_time = time.time() + + filepath = self._get_start_time_filename(seed) + + if not isinstance(start_time, float): + raise ValueError("Start time must be a float, but is %s." % type(start_time)) + + if os.path.exists(filepath): + raise ValueError( + "{filepath} already exist. Different seeds should be provided for different jobs." + ) + + with tempfile.NamedTemporaryFile('w', dir=os.path.dirname(filepath), delete=False) as fh: + fh.write(str(start_time)) + tempname = fh.name + os.rename(tempname, filepath) + + return filepath + + def load_start_time(self, seed: int) -> float: + with open(self._get_start_time_filename(seed), 'r') as fh: + start_time = float(fh.read()) + return start_time + + def get_smac_output_directory(self) -> str: + return os.path.join(self.temporary_directory, 'smac3-output') + + def get_smac_output_directory_for_run(self, seed: int) -> str: + return os.path.join( + self.temporary_directory, + 'smac3-output', + 'run_%d' % seed + ) + + def _get_targets_ensemble_filename(self) -> str: + return os.path.join(self.internals_directory, + "true_targets_ensemble.npy") + + def save_targets_ensemble(self, targets: np.ndarray) -> str: + self._make_internals_directory() + if not isinstance(targets, np.ndarray): + raise ValueError('Targets must be of type np.ndarray, but is %s' % + type(targets)) + + filepath = self._get_targets_ensemble_filename() + + # Try to open the file without locking it, this will reduce the + # number of times where we erroneously keep a lock on the ensemble + # targets file although the process already was killed + try: + existing_targets = np.load(filepath, allow_pickle=True) + if existing_targets.shape[0] > targets.shape[0] or ( + existing_targets.shape == targets.shape and np.allclose(existing_targets, targets)): + return filepath + except Exception: + pass + + with lockfile.LockFile(filepath): + if os.path.exists(filepath): + with open(filepath, 'rb') as fh: + existing_targets = np.load(fh, allow_pickle=True) + if existing_targets.shape[0] > targets.shape[0] or ( + existing_targets.shape == targets.shape and np.allclose(existing_targets, targets)): + return filepath + + with tempfile.NamedTemporaryFile('wb', dir=os.path.dirname( + filepath), delete=False) as fh_w: + np.save(fh_w, targets.astype(np.float32)) + tempname = fh_w.name + + os.rename(tempname, filepath) + + return filepath + + def load_targets_ensemble(self) -> np.ndarray: + filepath = self._get_targets_ensemble_filename() + + with lockfile.LockFile(filepath): + with open(filepath, 'rb') as fh: + targets = np.load(fh, allow_pickle=True) + + return targets + + def _get_datamanager_pickle_filename(self) -> str: + return os.path.join(self.internals_directory, 'datamanager.pkl') + + def save_datamanager(self, datamanager: BaseDataset) -> str: + self._make_internals_directory() + filepath = self._get_datamanager_pickle_filename() + + with lockfile.LockFile(filepath): + if not os.path.exists(filepath): + with tempfile.NamedTemporaryFile('wb', dir=os.path.dirname( + filepath), delete=False) as fh: + pickle.dump(datamanager, fh, -1) + tempname = fh.name + os.rename(tempname, filepath) + + return filepath + + def load_datamanager(self) -> BaseDataset: + filepath = self._get_datamanager_pickle_filename() + with lockfile.LockFile(filepath): + with open(filepath, 'rb') as fh: + return pickle.load(fh) + + def replace_datamanager(self, datamanager: BaseDataset) -> None: + """ + This function is called to replace the old datamanager with a datamanager + in case it is required. + + Args: + datamanager (BaseDataset): the new datamanager to replace the old. + """ + warnings.warn("Original dataset will be overwritten with the provided dataset") + datamanager_pickle_file = self._get_datamanager_pickle_filename() + if os.path.exists(datamanager_pickle_file): + os.remove(datamanager_pickle_file) + self.save_datamanager(datamanager=datamanager) + + def get_runs_directory(self) -> str: + return os.path.join(self.internals_directory, 'runs') + + def get_numrun_directory(self, seed: int, num_run: int, budget: float) -> str: + return os.path.join(self.internals_directory, 'runs', '%d_%d_%s' % (seed, num_run, budget)) + + def get_next_num_run(self, peek: bool = False) -> int: + """ + Every pipeline that is fitted by the estimator is stored with an + identifier called num_run. A dummy classifier will always have a num_run + equal to 1, and all other new configurations that are explored will + have a sequentially increasing identifier. + + This method returns the next num_run a configuration should take. + + Parameters + ---------- + peek: bool + By default, the next num_rum will be returned, i.e. self.active_num_run + 1 + Yet, if this bool parameter is equal to True, the value of the current + num_run is provided, i.e, self.active_num_run. + In other words, peek allows to get the current maximum identifier + of a configuration. + + Returns + ------- + num_run: int + An unique identifier for a configuration + """ + + # If there are other num_runs, their name would be runs/__ + other_num_runs = [int(os.path.basename(run_dir).split('_')[1]) + for run_dir in glob.glob(os.path.join(self.internals_directory, 'runs', '*')) + if re.match(r"\d+_\d+_\d+", os.path.basename(run_dir))] + if len(other_num_runs) > 0: + # We track the number of runs from two forefronts: + # The physically available num_runs (which might be deleted or a crash could happen) + # From a internally kept attribute. The later should be sufficient, but we + # want to be robust against multiple backend copies on different workers + self.active_num_run = max([self.active_num_run] + other_num_runs) + + # We are interested in the next run id + if not peek: + self.active_num_run += 1 + return self.active_num_run + + def get_model_filename(self, seed: int, idx: int, budget: float) -> str: + return '%s.%s.%s.model' % (seed, idx, budget) + + def get_cv_model_filename(self, seed: int, idx: int, budget: float) -> str: + return '%s.%s.%s.cv_model' % (seed, idx, budget) + + def list_all_models(self, seed: int) -> List[str]: + runs_directory = self.get_runs_directory() + model_files = glob.glob( + os.path.join(glob.escape(runs_directory), '%d_*' % seed, '%s.*.*.model' % seed) + ) + return model_files + + def load_models_by_identifiers(self, identifiers: List[Tuple[int, int, float]] + ) -> Dict: + models = dict() + + for identifier in identifiers: + seed, idx, budget = identifier + models[identifier] = self.load_model_by_seed_and_id_and_budget( + seed, idx, budget) + + return models + + def load_model_by_seed_and_id_and_budget(self, seed: int, + idx: int, + budget: float + ) -> BasePipeline: + model_directory = self.get_numrun_directory(seed, idx, budget) + + model_file_name = '%s.%s.%s.model' % (seed, idx, budget) + model_file_path = os.path.join(model_directory, model_file_name) + with open(model_file_path, 'rb') as fh: + return pickle.load(fh) + + def load_cv_models_by_identifiers(self, identifiers: List[Tuple[int, int, float]] + ) -> Dict: + models = dict() + + for identifier in identifiers: + seed, idx, budget = identifier + models[identifier] = self.load_cv_model_by_seed_and_id_and_budget( + seed, idx, budget) + + return models + + def load_cv_model_by_seed_and_id_and_budget(self, + seed: int, + idx: int, + budget: float + ) -> BasePipeline: + model_directory = self.get_numrun_directory(seed, idx, budget) + + model_file_name = '%s.%s.%s.cv_model' % (seed, idx, budget) + model_file_path = os.path.join(model_directory, model_file_name) + with open(model_file_path, 'rb') as fh: + return pickle.load(fh) + + def save_numrun_to_dir( + self, seed: int, idx: int, budget: float, model: Optional[BasePipeline], + cv_model: Optional[BasePipeline], ensemble_predictions: Optional[np.ndarray], + valid_predictions: Optional[np.ndarray], test_predictions: Optional[np.ndarray], + ) -> None: + assert self._logger is not None + runs_directory = self.get_runs_directory() + tmpdir = tempfile.mkdtemp(dir=runs_directory) + if model is not None: + file_path = os.path.join(tmpdir, self.get_model_filename(seed, idx, budget)) + with open(file_path, 'wb') as fh: + pickle.dump(model, fh, -1) + + if cv_model is not None: + file_path = os.path.join(tmpdir, self.get_cv_model_filename(seed, idx, budget)) + with open(file_path, 'wb') as fh: + pickle.dump(cv_model, fh, -1) + + for preds, subset in ( + (ensemble_predictions, 'ensemble'), + (valid_predictions, 'valid'), + (test_predictions, 'test') + ): + if preds is not None: + file_path = os.path.join( + tmpdir, + self.get_prediction_filename(subset, seed, idx, budget) + ) + with open(file_path, 'wb') as fh: + pickle.dump(preds.astype(np.float32), fh, -1) + try: + self._logger.debug("Renaming {} to {}".format(tmpdir, + self.get_numrun_directory(seed, idx, budget))) + os.rename(tmpdir, self.get_numrun_directory(seed, idx, budget)) + except OSError: + if os.path.exists(self.get_numrun_directory(seed, idx, budget)): + os.rename(self.get_numrun_directory(seed, idx, budget), + os.path.join(runs_directory, tmpdir + '.old')) + os.rename(tmpdir, self.get_numrun_directory(seed, idx, budget)) + shutil.rmtree(os.path.join(runs_directory, tmpdir + '.old')) + + def get_ensemble_dir(self) -> str: + return os.path.join(self.internals_directory, 'ensembles') + + def load_ensemble(self, seed: int) -> Optional[AbstractEnsemble]: + ensemble_dir = self.get_ensemble_dir() + + if not os.path.exists(ensemble_dir): + if self._logger is not None: + self._logger.warning('Directory %s does not exist' % ensemble_dir) + else: + warnings.warn('Directory %s does not exist' % ensemble_dir) + return None + + if seed >= 0: + indices_files = glob.glob( + os.path.join(glob.escape(ensemble_dir), '%s.*.ensemble' % seed) + ) + indices_files.sort() + else: + indices_files = os.listdir(ensemble_dir) + indices_files = [os.path.join(ensemble_dir, f) for f in indices_files] + indices_files.sort(key=lambda f: time.ctime(os.path.getmtime(f))) + + with open(indices_files[-1], 'rb') as fh: + ensemble_members_run_numbers = pickle.load(fh) + + return ensemble_members_run_numbers + + def save_ensemble(self, ensemble: AbstractEnsemble, idx: int, seed: int) -> None: + try: + os.makedirs(self.get_ensemble_dir()) + except Exception: + pass + + filepath = os.path.join( + self.get_ensemble_dir(), + '%s.%s.ensemble' % (str(seed), str(idx).zfill(10)) + ) + with tempfile.NamedTemporaryFile('wb', dir=os.path.dirname( + filepath), delete=False) as fh: + pickle.dump(ensemble, fh) + tempname = fh.name + os.rename(tempname, filepath) + + def get_prediction_filename(self, subset: str, + automl_seed: Union[str, int], + idx: int, + budget: float + ) -> str: + return 'predictions_%s_%s_%s_%s.npy' % (subset, automl_seed, idx, budget) + + def save_predictions_as_txt(self, + predictions: np.ndarray, + subset: str, + idx: int, precision: int, + prefix: Optional[str] = None) -> None: + if not self.output_directory: + return + # Write prediction scores in prescribed format + filepath = os.path.join( + self.output_directory, + ('%s_' % prefix if prefix else '') + '%s_%s.predict' % (subset, str(idx)), + ) + + format_string = '{:.%dg} ' % precision + with tempfile.NamedTemporaryFile('w', dir=os.path.dirname( + filepath), delete=False) as output_file: + for row in predictions: + if not isinstance(row, np.ndarray) and not isinstance(row, list): + row = [row] + for val in row: + output_file.write(format_string.format(float(val))) + output_file.write('\n') + tempname = output_file.name + os.rename(tempname, filepath) + + def write_txt_file(self, filepath: str, data: str, name: str) -> None: + with lockfile.LockFile(filepath): + with tempfile.NamedTemporaryFile('w', dir=os.path.dirname( + filepath), delete=False) as fh: + fh.write(data) + tempname = fh.name + os.rename(tempname, filepath) + if self._logger is not None: + self._logger.debug('Created %s file %s' % (name, filepath)) diff --git a/autoPyTorch/utils/common.py b/autoPyTorch/utils/common.py index a5082b1f3..552612c97 100644 --- a/autoPyTorch/utils/common.py +++ b/autoPyTorch/utils/common.py @@ -105,7 +105,31 @@ def __str__(self) -> str: return str(self.value) +<<<<<<< HEAD def custom_collate_fn(batch: List, x_collector: Callable = default_collate) -> List[Optional[torch.Tensor]]: +======= +def replace_prefix_in_config_dict(config: Dict[str, Any], prefix: str, replace: str = "") -> Dict[str, Any]: + """ + Replace the prefix in all keys with the specified replacement string (the empty string by + default to remove the prefix from the key). The functions makes sure that the prefix is a proper config + prefix by checking if it ends with ":", if not it appends ":" to the prefix. + + :param config: config dictionary where the prefixed of the keys should be replaced + :param prefix: prefix to be replaced in each key + :param replace: the string to replace the prefix with + :return: updated config dictionary + """ + # make sure that prefix ends with the config separator ":" + if not prefix.endswith(":"): + prefix = prefix + ":" + # only replace first occurrence of the prefix + return {k.replace(prefix, replace, 1): v + for k, v in config.items() if + k.startswith(prefix)} + + +def custom_collate_fn(batch: List) -> List[Optional[torch.Tensor]]: +>>>>>>> Bug fixes (#249) """ In the case of not providing a y tensor, in a dataset of form {X, y}, y would be None. diff --git a/examples/40_advanced/40_advanced/example_custom_configuration_space.py b/examples/40_advanced/40_advanced/example_custom_configuration_space.py new file mode 100644 index 000000000..b95ceeaa5 --- /dev/null +++ b/examples/40_advanced/40_advanced/example_custom_configuration_space.py @@ -0,0 +1,141 @@ +""" +====================== +Tabular Classification with Custom Configuration Space +====================== + +The following example shows how adjust the configuration space of +the search. Currently, there are two changes that can be made to the space:- +1. Adjust individual hyperparameters in the pipeline +2. Include or exclude components: + a) include: Dictionary containing components to include. Key is the node + name and Value is an Iterable of the names of the components + to include. Only these components will be present in the + search space. + b) exclude: Dictionary containing components to exclude. Key is the node + name and Value is an Iterable of the names of the components + to exclude. All except these components will be present in + the search space. +""" +import os +import tempfile as tmp +import warnings + +os.environ['JOBLIB_TEMP_FOLDER'] = tmp.gettempdir() +os.environ['OMP_NUM_THREADS'] = '1' +os.environ['OPENBLAS_NUM_THREADS'] = '1' +os.environ['MKL_NUM_THREADS'] = '1' + +warnings.simplefilter(action='ignore', category=UserWarning) +warnings.simplefilter(action='ignore', category=FutureWarning) + +import sklearn.datasets +import sklearn.model_selection + +from autoPyTorch.api.tabular_classification import TabularClassificationTask +from autoPyTorch.utils.hyperparameter_search_space_update import HyperparameterSearchSpaceUpdates + + +def get_search_space_updates(): + """ + Search space updates to the task can be added using HyperparameterSearchSpaceUpdates + Returns: + HyperparameterSearchSpaceUpdates + """ + updates = HyperparameterSearchSpaceUpdates() + updates.append(node_name="data_loader", + hyperparameter="batch_size", + value_range=[16, 512], + default_value=32) + updates.append(node_name="lr_scheduler", + hyperparameter="CosineAnnealingLR:T_max", + value_range=[50, 60], + default_value=55) + updates.append(node_name='network_backbone', + hyperparameter='ResNetBackbone:dropout', + value_range=[0, 0.5], + default_value=0.2) + updates.append(node_name='network_backbone', + hyperparameter='ResNetBackbone:multi_branch_choice', + value_range=['shake-shake'], + default_value='shake-shake') + updates.append(node_name='network_backbone', + hyperparameter='ResNetBackbone:shake_shake_method', + value_range=['M3'], + default_value='M3' + ) + return updates + + +if __name__ == '__main__': + + ############################################################################ + # Data Loading + # ============ + X, y = sklearn.datasets.fetch_openml(data_id=40981, return_X_y=True, as_frame=True) + X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split( + X, + y, + random_state=1, + ) + + ############################################################################ + # Build and fit a classifier with include components + # ================================================== + api = TabularClassificationTask( + search_space_updates=get_search_space_updates(), + include_components={'network_backbone': ['ResNetBackbone'], + 'encoder': ['OneHotEncoder']} + ) + + ############################################################################ + # Search for an ensemble of machine learning algorithms + # ===================================================== + api.search( + X_train=X_train.copy(), + y_train=y_train.copy(), + X_test=X_test.copy(), + y_test=y_test.copy(), + optimize_metric='accuracy', + total_walltime_limit=300, + func_eval_time_limit_secs=50 + ) + + ############################################################################ + # Print the final ensemble performance + # ==================================== + print(api.run_history, api.trajectory) + y_pred = api.predict(X_test) + score = api.score(y_pred, y_test) + print(score) + print(api.show_models()) + + ############################################################################ + # Build and fit a classifier with exclude components + # ================================================== + api = TabularClassificationTask( + search_space_updates=get_search_space_updates(), + exclude_components={'network_backbone': ['MLPBackbone'], + 'encoder': ['OneHotEncoder']} + ) + + ############################################################################ + # Search for an ensemble of machine learning algorithms + # ===================================================== + api.search( + X_train=X_train, + y_train=y_train, + X_test=X_test.copy(), + y_test=y_test.copy(), + optimize_metric='accuracy', + total_walltime_limit=300, + func_eval_time_limit_secs=50 + ) + + ############################################################################ + # Print the final ensemble performance + # ==================================== + print(api.run_history, api.trajectory) + y_pred = api.predict(X_test) + score = api.score(y_pred, y_test) + print(score) + print(api.show_models()) diff --git a/examples/40_advanced/40_advanced/example_posthoc_ensemble_fit.py b/examples/40_advanced/40_advanced/example_posthoc_ensemble_fit.py new file mode 100644 index 000000000..b9383b2a6 --- /dev/null +++ b/examples/40_advanced/40_advanced/example_posthoc_ensemble_fit.py @@ -0,0 +1,81 @@ +""" +===================================================== +Tabular Classification with Post-Hoc Ensemble Fitting +===================================================== + +The following example shows how to fit a sample classification model +and create an ensemble post-hoc with AutoPyTorch +""" +import os +import tempfile as tmp +import warnings + +os.environ['JOBLIB_TEMP_FOLDER'] = tmp.gettempdir() +os.environ['OMP_NUM_THREADS'] = '1' +os.environ['OPENBLAS_NUM_THREADS'] = '1' +os.environ['MKL_NUM_THREADS'] = '1' + +warnings.simplefilter(action='ignore', category=UserWarning) +warnings.simplefilter(action='ignore', category=FutureWarning) + +import sklearn.datasets +import sklearn.model_selection + +from autoPyTorch.api.tabular_classification import TabularClassificationTask + + +if __name__ == '__main__': + + ############################################################################ + # Data Loading + # ============ + X, y = sklearn.datasets.fetch_openml(data_id=40981, return_X_y=True, as_frame=True) + X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split( + X, + y, + random_state=42, + ) + + ############################################################################ + # Build and fit a classifier + # ========================== + api = TabularClassificationTask( + ensemble_size=0, + seed=42, + ) + + ############################################################################ + # Search for the best neural network + # ================================== + api.search( + X_train=X_train, + y_train=y_train, + X_test=X_test.copy(), + y_test=y_test.copy(), + optimize_metric='accuracy', + total_walltime_limit=250, + func_eval_time_limit_secs=50 + ) + + ############################################################################ + # Print the final performance of the incumbent neural network + # =========================================================== + print(api.run_history, api.trajectory) + y_pred = api.predict(X_test) + score = api.score(y_pred, y_test) + print(score) + + ############################################################################ + # Fit an ensemble with the neural networks fitted during the search + # ================================================================= + + api.fit_ensemble(ensemble_size=5, + # Set the enable_traditional_pipeline=True + # to also include traditional models + # in the ensemble + enable_traditional_pipeline=False) + # Print the final ensemble built by AutoPyTorch + y_pred = api.predict(X_test) + score = api.score(y_pred, y_test) + print(score) + print(api.show_models()) diff --git a/test/test_data/test_feature_validator.py b/test/test_data/test_feature_validator.py index d4a70c01c..122f3cc82 100644 --- a/test/test_data/test_feature_validator.py +++ b/test/test_data/test_feature_validator.py @@ -139,9 +139,9 @@ def test_featurevalidator_fitontypeA_transformtypeB(input_data_featuretest): if isinstance(input_data_featuretest, pd.DataFrame): pytest.skip("Column order change in pandas is not supported") elif isinstance(input_data_featuretest, np.ndarray): - complementary_type = validator.numpy_array_to_pandas(input_data_featuretest) + complementary_type = validator.numpy_to_pandas(input_data_featuretest) elif isinstance(input_data_featuretest, list): - complementary_type, _ = validator.list_to_dataframe(input_data_featuretest) + complementary_type, _ = validator.list_to_pandas(input_data_featuretest) elif sparse.issparse(input_data_featuretest): complementary_type = sparse.csr_matrix(input_data_featuretest.todense()) else: @@ -167,10 +167,128 @@ def test_featurevalidator_get_columns_to_encode(): for col in df.columns: df[col] = df[col].astype(col) +<<<<<<< HEAD transformed_columns, feature_types = validator._get_columns_to_encode(df) assert transformed_columns == ['category', 'bool'] assert feature_types == ['numerical', 'numerical', 'categorical', 'categorical'] +======= + validator.fit(df) + + categorical_columns, numerical_columns, feat_type = validator._get_columns_info(df) + + assert numerical_columns == ['int', 'float'] + assert categorical_columns == ['category', 'bool'] + assert feat_type == ['numerical', 'numerical', 'categorical', 'categorical'] + + +def feature_validator_remove_nan_catcolumns(df_train: pd.DataFrame, df_test: pd.DataFrame, + ans_train: np.ndarray, ans_test: np.ndarray) -> None: + validator = TabularFeatureValidator() + validator.fit(df_train) + transformed_df_train = validator.transform(df_train) + transformed_df_test = validator.transform(df_test) + + assert np.array_equal(transformed_df_train, ans_train) + assert np.array_equal(transformed_df_test, ans_test) + + +def test_feature_validator_remove_nan_catcolumns(): + """ + Make sure categorical columns that have only nan values are removed. + Transform performs the folloing: + * simple imputation for both + * scaling for numerical + * one-hot encoding for categorical + For example, + data = [ + {'A': 1, 'B': np.nan, 'C': np.nan}, + {'A': np.nan, 'B': 3, 'C': np.nan}, + {'A': 2, 'B': np.nan, 'C': np.nan} + ] + and suppose all the columns are categorical, + then + * `A` in {np.nan, 1, 2} + * `B` in {np.nan, 3} + * `C` in {np.nan} <=== it will be dropped. + + So in the column A, + * np.nan ==> [1, 0, 0] + * 1 ==> [0, 1, 0] + * 2 ==> [0, 0, 1] + in the column B, + * np.nan ==> [1, 0] + * 3 ==> [0, 1] + Therefore, by concatenating, + * {'A': 1, 'B': np.nan, 'C': np.nan} ==> [0, 1, 0, 1, 0] + * {'A': np.nan, 'B': 3, 'C': np.nan} ==> [1, 0, 0, 0, 1] + * {'A': 2, 'B': np.nan, 'C': np.nan} ==> [0, 0, 1, 1, 0] + """ + # First case, there exist null columns (B and C) in the train set + # and a same column (C) are not all null for the test set. + + df_train = pd.DataFrame( + [ + {'A': 1, 'B': np.nan, 'C': np.nan}, + {'A': np.nan, 'C': np.nan}, + {'A': 1} + ], + dtype='category', + ) + ans_train = np.array([[0, 1], [1, 0], [0, 1]], dtype=np.float64) + df_test = pd.DataFrame( + [ + {'A': np.nan, 'B': np.nan, 'C': 5}, + {'A': np.nan, 'C': np.nan}, + {'A': 1} + ], + dtype='category', + ) + ans_test = np.array([[1, 0], [1, 0], [0, 1]], dtype=np.float64) + feature_validator_remove_nan_catcolumns(df_train, df_test, ans_train, ans_test) + + # Second case, there exist null columns (B and C) in the training set and + # the same columns (B and C) are null in the test set. + df_train = pd.DataFrame( + [ + {'A': 1, 'B': np.nan, 'C': np.nan}, + {'A': np.nan, 'C': np.nan}, + {'A': 1} + ], + dtype='category', + ) + ans_train = np.array([[0, 1], [1, 0], [0, 1]], dtype=np.float64) + df_test = pd.DataFrame( + [ + {'A': np.nan, 'B': np.nan, 'C': np.nan}, + {'A': np.nan, 'C': np.nan}, + {'A': 1} + ], + dtype='category', + ) + ans_test = np.array([[1, 0], [1, 0], [0, 1]], dtype=np.float64) + feature_validator_remove_nan_catcolumns(df_train, df_test, ans_train, ans_test) + + # Third case, there exist no null columns in the training set and + # null columns exist in the test set. + df_train = pd.DataFrame( + [ + {'A': 1, 'B': 1}, + {'A': 2, 'B': 2} + ], + dtype='category', + ) + ans_train = np.array([[1, 0, 1, 0], [0, 1, 0, 1]], dtype=np.float64) + df_test = pd.DataFrame( + [ + {'A': np.nan, 'B': np.nan}, + {'A': np.nan, 'B': np.nan} + ], + dtype='category', + ) + ans_test = np.array([[0, 0, 0, 0], [0, 0, 0, 0]], dtype=np.float64) + feature_validator_remove_nan_catcolumns(df_train, df_test, ans_train, ans_test) +>>>>>>> Bug fixes (#249) def test_features_unsupported_calls_are_raised(): @@ -552,15 +670,16 @@ def test_feature_validator_imbalanced_data(): validator.fit(X_train) train_feature_types = copy.deepcopy(validator.feat_type) - assert train_feature_types == ['numerical', 'numerical', 'numerical', 'numerical'] + assert train_feature_types == ['numerical'] # validator will throw an error if the column types are not the same transformed_X_test = validator.transform(X_test) transformed_X_test = pd.DataFrame(transformed_X_test) - null_columns = [] - for column in transformed_X_test.columns: - if transformed_X_test[column].isna().all(): - null_columns.append(column) - assert null_columns == [0, 2, 3] + assert sorted(validator.all_nan_columns) == sorted(['A', 'C', 'D']) + # as there are no categorical columns, we can make such an + # assertion. We only expect to drop the all nan columns + total_all_nan_columns = len(validator.all_nan_columns) + total_columns = len(validator.column_order) + assert total_columns - total_all_nan_columns == len(transformed_X_test.columns) # Columns with not all null values in the train split and # completely null on the test split. @@ -579,15 +698,38 @@ def test_feature_validator_imbalanced_data(): X_test = pd.DataFrame.from_dict(test_features) validator = TabularFeatureValidator() validator.fit(X_train) + train_feature_types = copy.deepcopy(validator.feat_type) assert train_feature_types == ['categorical', 'numerical', 'numerical'] transformed_X_test = validator.transform(X_test) transformed_X_test = pd.DataFrame(transformed_X_test) - null_columns = [] - for column in transformed_X_test.columns: - if transformed_X_test[column].isna().all(): - null_columns.append(column) + assert not len(validator.all_nan_columns) + +<<<<<<< HEAD assert null_columns == [1] >>>>>>> Fixing issues with imbalanced datasets (#197) +======= +def test_comparator(): + numerical = 'numerical' + categorical = 'categorical' + + validator = TabularFeatureValidator + + feat_type = [numerical, categorical] * 10 + ans = [categorical] * 10 + [numerical] * 10 + feat_type = sorted( + feat_type, + key=functools.cmp_to_key(validator._comparator) + ) + assert ans == feat_type + + feat_type = [numerical] * 10 + [categorical] * 10 + ans = [categorical] * 10 + [numerical] * 10 + feat_type = sorted( + feat_type, + key=functools.cmp_to_key(validator._comparator) + ) + assert ans == feat_type +>>>>>>> Bug fixes (#249) From 392f07a4beb248c1170456b6dcc30f9b61c1d1fd Mon Sep 17 00:00:00 2001 From: Ravin Kohli <13005107+ravinkohli@users.noreply.github.com> Date: Tue, 7 Dec 2021 21:06:27 +0100 Subject: [PATCH 37/50] [FIX] Passing checks (#298) * Initial fix for all tests passing locally py=3.8 * fix bug in tests * fix bug in test for data * debugging error in dummy forward pass * debug try -2 * catch runtime error in ci * catch runtime error in ci * add better debug test setup * debug some more * run this test only * remove sum backward * remove inplace in inception block * undo silly change * Enable all tests * fix flake * fix bug in test setup * remove anamoly detection * minor changes to comments * Apply suggestions from code review Co-authored-by: nabenabe0928 <47781922+nabenabe0928@users.noreply.github.com> * Address comments from Shuhei * revert change leading to bug * fix flake * change comment position in feature validator * Add documentation for _is_datasets_consistent * address comments from arlind * case when all nans in test Co-authored-by: nabenabe0928 <47781922+nabenabe0928@users.noreply.github.com> --- autoPyTorch/api/base_task.py | 5 +- autoPyTorch/data/tabular_feature_validator.py | 50 ++++-- .../encoding/NoEncoder.py | 2 +- .../tabular_preprocessing/scaling/NoScaler.py | 2 +- .../base_network_embedding.py | 7 +- .../training/trainer/AdversarialTrainer.py | 6 +- .../components/training/trainer/__init__.py | 1 + .../example_custom_configuration_space.py | 2 +- test/test_data/test_feature_validator.py | 150 +++--------------- test/test_data/test_validation.py | 20 --- .../components/preprocessing/test_encoders.py | 2 + .../components/preprocessing/test_imputers.py | 2 + .../components/preprocessing/test_scalers.py | 8 + .../test_tabular_column_transformer.py | 2 + .../components/setup/test_setup_networks.py | 3 +- .../components/training/test_training.py | 9 +- .../test_tabular_classification.py | 16 +- test/test_pipeline/test_tabular_regression.py | 22 ++- 18 files changed, 109 insertions(+), 200 deletions(-) diff --git a/autoPyTorch/api/base_task.py b/autoPyTorch/api/base_task.py index 3a902878e..ad3696101 100644 --- a/autoPyTorch/api/base_task.py +++ b/autoPyTorch/api/base_task.py @@ -1773,7 +1773,7 @@ def fit_ensemble( Args: optimize_metric (str): name of the metric that is used to evaluate a pipeline. if not specified, value passed to search will be used - precision (int), (default=32): Numeric precision used when loading + precision (Optional[int]): Numeric precision used when loading ensemble data. Can be either 16, 32 or 64. ensemble_nbest (Optional[int]): only consider the ensemble_nbest models to build the ensemble. @@ -1816,6 +1816,7 @@ def fit_ensemble( "Please call the `search()` method of {} prior to " "fit_ensemble().".format(self.__class__.__name__)) + precision = precision if precision is not None else self.precision if precision not in [16, 32, 64]: raise ValueError("precision must be one of 16, 32, 64 but got {}".format(precision)) @@ -1866,7 +1867,7 @@ def fit_ensemble( manager = self._init_ensemble_builder( time_left_for_ensembles=time_left_for_ensemble, optimize_metric=self.opt_metric if optimize_metric is None else optimize_metric, - precision=self.precision if precision is None else precision, + precision=precision, ensemble_size=ensemble_size, ensemble_nbest=ensemble_nbest, ) diff --git a/autoPyTorch/data/tabular_feature_validator.py b/autoPyTorch/data/tabular_feature_validator.py index 2b4285402..ad305c588 100644 --- a/autoPyTorch/data/tabular_feature_validator.py +++ b/autoPyTorch/data/tabular_feature_validator.py @@ -139,6 +139,7 @@ def _comparator(cmp1: str, cmp2: str) -> int: if cmp1 not in choices or cmp2 not in choices: raise ValueError('The comparator for the column order only accepts {}, ' 'but got {} and {}'.format(choices, cmp1, cmp2)) + idx1, idx2 = choices.index(cmp1), choices.index(cmp2) return idx1 - idx2 @@ -284,13 +285,12 @@ def transform( # having a value for a categorical column. # We need to convert the column in test data to # object otherwise the test column is interpreted as float - if len(self.categorical_columns) > 0: - categorical_columns = self.column_transformer.transformers_[0][-1] - for column in categorical_columns: - if X[column].isna().all(): - X[column] = X[column].astype('object') - if self.column_transformer is not None: + if len(self.categorical_columns) > 0: + categorical_columns = self.column_transformer.transformers_[0][-1] + for column in categorical_columns: + if X[column].isna().all(): + X[column] = X[column].astype('object') X = self.column_transformer.transform(X) # Sparse related transformations @@ -379,16 +379,11 @@ def _check_data( self.column_order = column_order dtypes = [dtype.name for dtype in X.dtypes] - dtypes_diff = [s_dtype != dtype for s_dtype, dtype in zip(self.dtypes, dtypes)] + + diff_cols = X.columns[[s_dtype != dtype for s_dtype, dtype in zip(self.dtypes, dtypes)]] if len(self.dtypes) == 0: self.dtypes = dtypes - elif ( - any(dtypes_diff) # the dtypes of some columns are different in train and test dataset - and self.all_nan_columns is not None # Ignore all_nan_columns is None - and len(set(X.columns[dtypes_diff]).difference(self.all_nan_columns)) != 0 - ): - # The dtypes can be different if and only if the column belongs - # to all_nan_columns as these columns would be imputed. + elif not self._is_datasets_consistent(diff_cols, X): raise ValueError("The dtype of the features must not be changed after fit(), but" " the dtypes of some columns are different between training ({}) and" " test ({}) datasets.".format(self.dtypes, dtypes)) @@ -619,6 +614,33 @@ def infer_objects(self, X: pd.DataFrame) -> pd.DataFrame: return X + def _is_datasets_consistent(self, diff_cols: List[Union[int, str]], X: pd.DataFrame) -> bool: + """ + Check the consistency of dtypes between training and test datasets. + The dtypes can be different if the column belongs to `self.all_nan_columns` + (list of column names with all nans in training data) or if the column is + all nan as these columns would be imputed. + + Args: + diff_cols (List[bool]): + The column labels that have different dtypes. + X (pd.DataFrame): + A validation or test dataset to be compared with the training dataset + Returns: + _ (bool): Whether the training and test datasets are consistent. + """ + if self.all_nan_columns is None: + if len(diff_cols) == 0: + return True + else: + return all(X[diff_cols].isna().all()) + + # dtype is different ==> the column in at least either of train or test datasets must be all NaN + # inconsistent <==> dtype is different and the col in both train and test is not all NaN + inconsistent_cols = list(set(diff_cols) - self.all_nan_columns) + + return len(inconsistent_cols) == 0 or all(X[inconsistent_cols].isna().all()) + def has_object_columns( feature_types: pd.Series, diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/encoding/NoEncoder.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/encoding/NoEncoder.py index 929e99048..d62ee26d2 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/encoding/NoEncoder.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/encoding/NoEncoder.py @@ -40,7 +40,7 @@ def transform(self, X: Dict[str, Any]) -> Dict[str, Any]: Returns: (Dict[str, Any]): the updated 'X' dictionary """ - X.update({'encoder': self.preprocessor}) + # X.update({'encoder': self.preprocessor}) return X @staticmethod diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/NoScaler.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/NoScaler.py index 9d50aa8f5..9775d17dd 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/NoScaler.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/NoScaler.py @@ -43,7 +43,7 @@ def transform(self, X: Dict[str, Any]) -> Dict[str, Any]: Returns: np.ndarray: Transformed features """ - X.update({'scaler': self.preprocessor}) + # X.update({'scaler': self.preprocessor}) return X @staticmethod diff --git a/autoPyTorch/pipeline/components/setup/network_embedding/base_network_embedding.py b/autoPyTorch/pipeline/components/setup/network_embedding/base_network_embedding.py index 4261cad84..334677f49 100644 --- a/autoPyTorch/pipeline/components/setup/network_embedding/base_network_embedding.py +++ b/autoPyTorch/pipeline/components/setup/network_embedding/base_network_embedding.py @@ -1,10 +1,6 @@ -<<<<<<< HEAD import copy from typing import Any, Dict, List, Optional, Tuple, Union -======= -# import copy -from typing import Any, Dict, Optional, Tuple ->>>>>>> Bug fixes (#249) + import numpy as np @@ -40,6 +36,7 @@ def fit(self, X: Dict[str, Any], y: Any = None) -> BaseEstimator: self.feature_shapes = feature_shapes else: self.feature_shapes = X['dataset_properties']['feature_shapes'] + return self def transform(self, X: Dict[str, Any]) -> Dict[str, Any]: diff --git a/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py b/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py index 7f5385382..0fefd9525 100644 --- a/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py @@ -109,11 +109,7 @@ def train_step(self, data: np.ndarray, targets: np.ndarray) -> Tuple[float, torc loss = loss_func(self.criterion, original_outputs, adversarial_outputs) loss.backward() self.optimizer.step() - if self.scheduler: - if 'ReduceLROnPlateau' in self.scheduler.__class__.__name__: - self.scheduler.step(loss) - else: - self.scheduler.step() + # only passing the original outputs since we do not care about # the adversarial performance. return loss.item(), original_outputs diff --git a/autoPyTorch/pipeline/components/training/trainer/__init__.py b/autoPyTorch/pipeline/components/training/trainer/__init__.py index 282d356b8..b740b8b53 100755 --- a/autoPyTorch/pipeline/components/training/trainer/__init__.py +++ b/autoPyTorch/pipeline/components/training/trainer/__init__.py @@ -282,6 +282,7 @@ def fit(self, X: Dict[str, Any], y: Any = None, **kwargs: Any) -> autoPyTorchCom y=y, **kwargs ) + # Add snapshots to base network to enable # predicting with snapshot ensemble self.choice: autoPyTorchComponent = cast(autoPyTorchComponent, self.choice) diff --git a/examples/40_advanced/40_advanced/example_custom_configuration_space.py b/examples/40_advanced/40_advanced/example_custom_configuration_space.py index b95ceeaa5..25eb86be7 100644 --- a/examples/40_advanced/40_advanced/example_custom_configuration_space.py +++ b/examples/40_advanced/40_advanced/example_custom_configuration_space.py @@ -59,7 +59,7 @@ def get_search_space_updates(): value_range=['shake-shake'], default_value='shake-shake') updates.append(node_name='network_backbone', - hyperparameter='ResNetBackbone:shake_shake_method', + hyperparameter='ResNetBackbone:shake_shake_update_func', value_range=['M3'], default_value='M3' ) diff --git a/test/test_data/test_feature_validator.py b/test/test_data/test_feature_validator.py index 122f3cc82..bb4193bdf 100644 --- a/test/test_data/test_feature_validator.py +++ b/test/test_data/test_feature_validator.py @@ -1,4 +1,4 @@ - import copy +import copy import functools import numpy as np @@ -139,9 +139,9 @@ def test_featurevalidator_fitontypeA_transformtypeB(input_data_featuretest): if isinstance(input_data_featuretest, pd.DataFrame): pytest.skip("Column order change in pandas is not supported") elif isinstance(input_data_featuretest, np.ndarray): - complementary_type = validator.numpy_to_pandas(input_data_featuretest) + complementary_type = validator.numpy_array_to_pandas(input_data_featuretest) elif isinstance(input_data_featuretest, list): - complementary_type, _ = validator.list_to_pandas(input_data_featuretest) + complementary_type, _ = validator.list_to_dataframe(input_data_featuretest) elif sparse.issparse(input_data_featuretest): complementary_type = sparse.csr_matrix(input_data_featuretest.todense()) else: @@ -167,128 +167,10 @@ def test_featurevalidator_get_columns_to_encode(): for col in df.columns: df[col] = df[col].astype(col) -<<<<<<< HEAD transformed_columns, feature_types = validator._get_columns_to_encode(df) assert transformed_columns == ['category', 'bool'] assert feature_types == ['numerical', 'numerical', 'categorical', 'categorical'] -======= - validator.fit(df) - - categorical_columns, numerical_columns, feat_type = validator._get_columns_info(df) - - assert numerical_columns == ['int', 'float'] - assert categorical_columns == ['category', 'bool'] - assert feat_type == ['numerical', 'numerical', 'categorical', 'categorical'] - - -def feature_validator_remove_nan_catcolumns(df_train: pd.DataFrame, df_test: pd.DataFrame, - ans_train: np.ndarray, ans_test: np.ndarray) -> None: - validator = TabularFeatureValidator() - validator.fit(df_train) - transformed_df_train = validator.transform(df_train) - transformed_df_test = validator.transform(df_test) - - assert np.array_equal(transformed_df_train, ans_train) - assert np.array_equal(transformed_df_test, ans_test) - - -def test_feature_validator_remove_nan_catcolumns(): - """ - Make sure categorical columns that have only nan values are removed. - Transform performs the folloing: - * simple imputation for both - * scaling for numerical - * one-hot encoding for categorical - For example, - data = [ - {'A': 1, 'B': np.nan, 'C': np.nan}, - {'A': np.nan, 'B': 3, 'C': np.nan}, - {'A': 2, 'B': np.nan, 'C': np.nan} - ] - and suppose all the columns are categorical, - then - * `A` in {np.nan, 1, 2} - * `B` in {np.nan, 3} - * `C` in {np.nan} <=== it will be dropped. - - So in the column A, - * np.nan ==> [1, 0, 0] - * 1 ==> [0, 1, 0] - * 2 ==> [0, 0, 1] - in the column B, - * np.nan ==> [1, 0] - * 3 ==> [0, 1] - Therefore, by concatenating, - * {'A': 1, 'B': np.nan, 'C': np.nan} ==> [0, 1, 0, 1, 0] - * {'A': np.nan, 'B': 3, 'C': np.nan} ==> [1, 0, 0, 0, 1] - * {'A': 2, 'B': np.nan, 'C': np.nan} ==> [0, 0, 1, 1, 0] - """ - # First case, there exist null columns (B and C) in the train set - # and a same column (C) are not all null for the test set. - - df_train = pd.DataFrame( - [ - {'A': 1, 'B': np.nan, 'C': np.nan}, - {'A': np.nan, 'C': np.nan}, - {'A': 1} - ], - dtype='category', - ) - ans_train = np.array([[0, 1], [1, 0], [0, 1]], dtype=np.float64) - df_test = pd.DataFrame( - [ - {'A': np.nan, 'B': np.nan, 'C': 5}, - {'A': np.nan, 'C': np.nan}, - {'A': 1} - ], - dtype='category', - ) - ans_test = np.array([[1, 0], [1, 0], [0, 1]], dtype=np.float64) - feature_validator_remove_nan_catcolumns(df_train, df_test, ans_train, ans_test) - - # Second case, there exist null columns (B and C) in the training set and - # the same columns (B and C) are null in the test set. - df_train = pd.DataFrame( - [ - {'A': 1, 'B': np.nan, 'C': np.nan}, - {'A': np.nan, 'C': np.nan}, - {'A': 1} - ], - dtype='category', - ) - ans_train = np.array([[0, 1], [1, 0], [0, 1]], dtype=np.float64) - df_test = pd.DataFrame( - [ - {'A': np.nan, 'B': np.nan, 'C': np.nan}, - {'A': np.nan, 'C': np.nan}, - {'A': 1} - ], - dtype='category', - ) - ans_test = np.array([[1, 0], [1, 0], [0, 1]], dtype=np.float64) - feature_validator_remove_nan_catcolumns(df_train, df_test, ans_train, ans_test) - - # Third case, there exist no null columns in the training set and - # null columns exist in the test set. - df_train = pd.DataFrame( - [ - {'A': 1, 'B': 1}, - {'A': 2, 'B': 2} - ], - dtype='category', - ) - ans_train = np.array([[1, 0, 1, 0], [0, 1, 0, 1]], dtype=np.float64) - df_test = pd.DataFrame( - [ - {'A': np.nan, 'B': np.nan}, - {'A': np.nan, 'B': np.nan} - ], - dtype='category', - ) - ans_test = np.array([[0, 0, 0, 0], [0, 0, 0, 0]], dtype=np.float64) - feature_validator_remove_nan_catcolumns(df_train, df_test, ans_train, ans_test) ->>>>>>> Bug fixes (#249) def test_features_unsupported_calls_are_raised(): @@ -529,6 +411,7 @@ def test_comparator(): assert ans == feat_type +<<<<<<< HEAD @pytest.fixture def input_data_feature_feat_types(request): if request.param == 'pandas_categoricalonly': @@ -648,6 +531,8 @@ def test_feature_validator_get_columns_to_encode_error_feat_type(input_data_feat with pytest.raises(ValueError, match=r"Expected type of features to be in .*"): validator._validate_feat_types(X) +======= +>>>>>>> [FIX] Passing checks (#298) def test_feature_validator_imbalanced_data(): # Null columns in the train split but not necessarily in the test split @@ -670,16 +555,15 @@ def test_feature_validator_imbalanced_data(): validator.fit(X_train) train_feature_types = copy.deepcopy(validator.feat_type) - assert train_feature_types == ['numerical'] + assert train_feature_types == ['numerical', 'numerical', 'numerical', 'numerical'] # validator will throw an error if the column types are not the same transformed_X_test = validator.transform(X_test) transformed_X_test = pd.DataFrame(transformed_X_test) - assert sorted(validator.all_nan_columns) == sorted(['A', 'C', 'D']) - # as there are no categorical columns, we can make such an - # assertion. We only expect to drop the all nan columns - total_all_nan_columns = len(validator.all_nan_columns) - total_columns = len(validator.column_order) - assert total_columns - total_all_nan_columns == len(transformed_X_test.columns) + null_columns = [] + for column in transformed_X_test.columns: + if transformed_X_test[column].isna().all(): + null_columns.append(column) + assert null_columns == [0, 2, 3] # Columns with not all null values in the train split and # completely null on the test split. @@ -698,12 +582,12 @@ def test_feature_validator_imbalanced_data(): X_test = pd.DataFrame.from_dict(test_features) validator = TabularFeatureValidator() validator.fit(X_train) - train_feature_types = copy.deepcopy(validator.feat_type) assert train_feature_types == ['categorical', 'numerical', 'numerical'] transformed_X_test = validator.transform(X_test) transformed_X_test = pd.DataFrame(transformed_X_test) +<<<<<<< HEAD assert not len(validator.all_nan_columns) @@ -733,3 +617,11 @@ def test_comparator(): ) assert ans == feat_type >>>>>>> Bug fixes (#249) +======= + null_columns = [] + for column in transformed_X_test.columns: + if transformed_X_test[column].isna().all(): + null_columns.append(column) + + assert null_columns == [1] +>>>>>>> [FIX] Passing checks (#298) diff --git a/test/test_data/test_validation.py b/test/test_data/test_validation.py index 341900413..b58b29b59 100644 --- a/test/test_data/test_validation.py +++ b/test/test_data/test_validation.py @@ -1,7 +1,5 @@ import numpy as np -import pandas as pd - import pytest from scipy import sparse @@ -33,14 +31,6 @@ def test_data_validation_for_classification(openmlid, as_frame): validator.fit(X_train=X_train, y_train=y_train, X_test=X_test, y_test=y_test) X_train_t, y_train_t = validator.transform(X_train, y_train) - assert np.shape(X_train) == np.shape(X_train_t) - - # Leave columns that are complete NaN - # The sklearn pipeline will handle that - if as_frame and np.any(pd.isnull(X_train).values.all(axis=0)): - assert np.any(pd.isnull(X_train_t).values.all(axis=0)) - elif not as_frame and np.any(pd.isnull(X_train).all(axis=0)): - assert np.any(pd.isnull(X_train_t).all(axis=0)) # make sure everything was encoded to number assert np.issubdtype(X_train_t.dtype, np.number) @@ -75,14 +65,6 @@ def test_data_validation_for_regression(openmlid, as_frame): validator.fit(X_train=X_train, y_train=y_train) X_train_t, y_train_t = validator.transform(X_train, y_train) - assert np.shape(X_train) == np.shape(X_train_t) - - # Leave columns that are complete NaN - # The sklearn pipeline will handle that - if as_frame and np.any(pd.isnull(X_train).values.all(axis=0)): - assert np.any(pd.isnull(X_train_t).values.all(axis=0)) - elif not as_frame and np.any(pd.isnull(X_train).all(axis=0)): - assert np.any(pd.isnull(X_train_t).all(axis=0)) # make sure everything was encoded to number assert np.issubdtype(X_train_t.dtype, np.number) @@ -104,8 +86,6 @@ def test_sparse_data_validation_for_regression(): validator.fit(X_train=X_sp, y_train=y) X_t, y_t = validator.transform(X, y) - assert np.shape(X) == np.shape(X_t) - # make sure everything was encoded to number assert np.issubdtype(X_t.dtype, np.number) assert np.issubdtype(y_t.dtype, np.number) diff --git a/test/test_pipeline/components/preprocessing/test_encoders.py b/test/test_pipeline/components/preprocessing/test_encoders.py index a901823ba..ac796291c 100644 --- a/test/test_pipeline/components/preprocessing/test_encoders.py +++ b/test/test_pipeline/components/preprocessing/test_encoders.py @@ -10,6 +10,8 @@ from autoPyTorch.pipeline.components.preprocessing.tabular_preprocessing.encoding.OneHotEncoder import OneHotEncoder +# TODO: fix in preprocessing PR +@unittest.skip("Skipping tests as preprocessing is not finalised") class TestEncoders(unittest.TestCase): def test_one_hot_encoder_no_unknown(self): diff --git a/test/test_pipeline/components/preprocessing/test_imputers.py b/test/test_pipeline/components/preprocessing/test_imputers.py index 0db460b77..ad9ed710f 100644 --- a/test/test_pipeline/components/preprocessing/test_imputers.py +++ b/test/test_pipeline/components/preprocessing/test_imputers.py @@ -11,6 +11,8 @@ from autoPyTorch.pipeline.components.preprocessing.tabular_preprocessing.imputation.SimpleImputer import SimpleImputer +# TODO: fix in preprocessing PR +@unittest.skip("Skipping tests as preprocessing is not finalised") class TestSimpleImputer(unittest.TestCase): def test_get_config_space(self): diff --git a/test/test_pipeline/components/preprocessing/test_scalers.py b/test/test_pipeline/components/preprocessing/test_scalers.py index 7cbc12b07..8d05c8da1 100644 --- a/test/test_pipeline/components/preprocessing/test_scalers.py +++ b/test/test_pipeline/components/preprocessing/test_scalers.py @@ -17,6 +17,8 @@ from autoPyTorch.pipeline.components.preprocessing.tabular_preprocessing.scaling.StandardScaler import StandardScaler +# TODO: fix in preprocessing PR +@unittest.skip("Skipping tests as preprocessing is not finalised") class TestNormalizer(unittest.TestCase): def test_l2_norm(self): @@ -134,6 +136,8 @@ def test_max_norm(self): [0.84615385, 0.92307692, 1]])) +# TODO: fix in preprocessing PR +@unittest.skip("Skipping tests as preprocessing is not finalised") class TestMinMaxScaler(unittest.TestCase): def test_minmax_scaler(self): @@ -175,6 +179,8 @@ def test_minmax_scaler(self): [0.76923077, 0.76923077, 0.76923077]])) +# TODO: fix in preprocessing PR +@unittest.skip("Skipping tests as preprocessing is not finalised") class TestStandardScaler(unittest.TestCase): def test_standard_scaler(self): @@ -217,6 +223,8 @@ def test_standard_scaler(self): [0.8396642, 0.8396642, 0.8396642]])) +# TODO: fix in preprocessing PR +@unittest.skip("Skipping tests as preprocessing is not finalised") class TestNoneScaler(unittest.TestCase): def test_none_scaler(self): diff --git a/test/test_pipeline/components/preprocessing/test_tabular_column_transformer.py b/test/test_pipeline/components/preprocessing/test_tabular_column_transformer.py index 36de9f275..6db124be1 100644 --- a/test/test_pipeline/components/preprocessing/test_tabular_column_transformer.py +++ b/test/test_pipeline/components/preprocessing/test_tabular_column_transformer.py @@ -13,6 +13,8 @@ ) +# TODO: fix in preprocessing PR +@pytest.mark.skip("Skipping tests as preprocessing is not finalised") @pytest.mark.parametrize("fit_dictionary_tabular", ['classification_numerical_only', 'classification_categorical_only', 'classification_numerical_and_categorical'], indirect=True) diff --git a/test/test_pipeline/components/setup/test_setup_networks.py b/test/test_pipeline/components/setup/test_setup_networks.py index f3b9ff11c..f5e9b1bb7 100644 --- a/test/test_pipeline/components/setup/test_setup_networks.py +++ b/test/test_pipeline/components/setup/test_setup_networks.py @@ -19,7 +19,8 @@ def head(request): return request.param -@pytest.fixture(params=['LearnedEntityEmbedding', 'NoEmbedding']) +# TODO: add 'LearnedEntityEmbedding' after preprocessing dix +@pytest.fixture(params=['NoEmbedding']) def embedding(request): return request.param diff --git a/test/test_pipeline/components/training/test_training.py b/test/test_pipeline/components/training/test_training.py index 4cc4efe29..44a903308 100644 --- a/test/test_pipeline/components/training/test_training.py +++ b/test/test_pipeline/components/training/test_training.py @@ -395,7 +395,7 @@ def test_every_trainer_is_valid(): @pytest.mark.parametrize("test_input,expected", [ ("tabular_classification", set(['RowCutMixTrainer', 'RowCutOutTrainer', 'AdversarialTrainer'])), - ("image_classification", set(['GridCutMixTrainer', 'GridCutOutTrainer'])), + ("image_classification", set(['GridCutMixTrainer', 'GridCutOutTrainer', 'AdversarialTrainer'])), ("time_series_classification", set([])), ]) def test_get_set_config_space(test_input, expected): @@ -470,7 +470,7 @@ def criterion(a, b): (GridCutOutTrainer, torch.from_numpy(np.full(shape=(2, 3, 10, 12), fill_value=255))), (RowCutOutTrainer, torch.from_numpy(np.array([[1, 2, 3, 4, 5], [6, 7, 8, 9, 10]]))), ]) -def test_cutput_regularizers(cutout_prob, regularizer, X): +def test_cutout_regularizers(cutout_prob, regularizer, X): trainer = regularizer(cutout_prob=cutout_prob, patch_ratio=0.5) y = torch.from_numpy(np.array([[1], [0]])) @@ -483,10 +483,7 @@ def test_cutput_regularizers(cutout_prob, regularizer, X): np.testing.assert_array_equal(X_new.numpy(), X.numpy()) else: # There has to be a change in the features - if len(X.shape) > 2: - expected = 0.0 - else: - expected = -1 + expected = 0.0 # The original X does not have the expected value # If a cutoff happened, then this value is gonna be there assert expected in X_new diff --git a/test/test_pipeline/test_tabular_classification.py b/test/test_pipeline/test_tabular_classification.py index b0923fd41..29d5dfbaa 100644 --- a/test/test_pipeline/test_tabular_classification.py +++ b/test/test_pipeline/test_tabular_classification.py @@ -33,7 +33,7 @@ @pytest.fixture def exclude(): - return {'feature_preprocessor': ['SelectRatesClassification', 'SelectPercentileClassification']} + return {'feature_preprocessor': ['SelectRatesClassification', 'SelectPercentileClassification'], 'network_embedding': ['LearnedEntityEmbedding']} @pytest.mark.parametrize("fit_dictionary_tabular", ['classification_categorical_only', @@ -187,9 +187,11 @@ def test_pipeline_transform(self, fit_dictionary_tabular, exclude): assert fit_dictionary_tabular.items() <= transformed_fit_dictionary_tabular.items() # Then the pipeline should have added the following keys - expected_keys = {'imputer', 'encoder', 'scaler', 'tabular_transformer', - 'preprocess_transforms', 'network', 'optimizer', 'lr_scheduler', - 'train_data_loader', 'val_data_loader', 'run_summary'} + # Removing 'imputer', 'encoder', 'scaler', these will be + # added back after a PR fixing preprocessing + expected_keys = {'tabular_transformer', 'preprocess_transforms', 'network', + 'optimizer', 'lr_scheduler', 'train_data_loader', + 'val_data_loader', 'run_summary', 'feature_preprocessor'} assert expected_keys.issubset(set(transformed_fit_dictionary_tabular.keys())) # Then we need to have transformations being created. @@ -325,8 +327,8 @@ def test_error_search_space_updates(self, fit_dictionary_tabular, error_search_s search_space_updates=error_search_space_updates) except Exception as e: assert isinstance(e, ValueError) - assert re.match(r'Unknown hyperparameter for component .*?\. Expected update ' - r'hyperparameter to be in \[.*?\] got .+', e.args[0]) + assert re.match(r'Unknown hyperparameter for .*?\. Expected update ' + r'hyperparameter to be in \[.*?\], but got .+', e.args[0]) def test_set_range_search_space_updates(self, fit_dictionary_tabular): dataset_properties = {'numerical_columns': [1], 'categorical_columns': [2], @@ -399,7 +401,7 @@ def test_set_choices_updates(self, fit_dictionary_tabular): 'ReduceLROnPlateau']) def test_trainer_cocktails(self, fit_dictionary_tabular, mocker, lr_scheduler, trainer): # noqa F811 fit_dictionary_tabular['epochs'] = 45 - fit_dictionary_tabular['early_stopping'] = 20 + fit_dictionary_tabular['early_stopping'] = -1 pipeline = TabularClassificationPipeline( dataset_properties=fit_dictionary_tabular['dataset_properties'], include={'lr_scheduler': [lr_scheduler], 'trainer': [trainer]}) diff --git a/test/test_pipeline/test_tabular_regression.py b/test/test_pipeline/test_tabular_regression.py index c26be05b4..5889ed1c6 100644 --- a/test/test_pipeline/test_tabular_regression.py +++ b/test/test_pipeline/test_tabular_regression.py @@ -60,9 +60,11 @@ def test_pipeline_fit(self, fit_dictionary_tabular): """This test makes sure that the pipeline is able to fit given random combinations of hyperparameters across the pipeline""" # TODO: fix issue where adversarial also works for regression + # TODO: Fix issue with learned entity embedding after preprocessing PR pipeline = TabularRegressionPipeline( dataset_properties=fit_dictionary_tabular['dataset_properties'], - exclude={'trainer': ['AdversarialTrainer']}) + exclude={'trainer': ['AdversarialTrainer'], + 'network_embedding': ['LearnedEntityEmbedding']}) cs = pipeline.get_hyperparameter_search_space() config = cs.sample_configuration() @@ -88,7 +90,8 @@ def test_pipeline_predict(self, fit_dictionary_tabular): X = fit_dictionary_tabular['X_train'].copy() pipeline = TabularRegressionPipeline( dataset_properties=fit_dictionary_tabular['dataset_properties'], - exclude={'trainer': ['AdversarialTrainer']}) + exclude={'trainer': ['AdversarialTrainer'], + 'network_embedding': ['LearnedEntityEmbedding']}) cs = pipeline.get_hyperparameter_search_space() config = cs.sample_configuration() @@ -117,7 +120,8 @@ def test_pipeline_transform(self, fit_dictionary_tabular): pipeline = TabularRegressionPipeline( dataset_properties=fit_dictionary_tabular['dataset_properties'], - exclude={'trainer': ['AdversarialTrainer']}) + exclude={'trainer': ['AdversarialTrainer'], + 'network_embedding': ['LearnedEntityEmbedding']}) cs = pipeline.get_hyperparameter_search_space() config = cs.sample_configuration() pipeline.set_hyperparameters(config) @@ -134,9 +138,11 @@ def test_pipeline_transform(self, fit_dictionary_tabular): assert fit_dictionary_tabular.items() <= transformed_fit_dictionary_tabular.items() # Then the pipeline should have added the following keys - expected_keys = {'imputer', 'encoder', 'scaler', 'tabular_transformer', - 'preprocess_transforms', 'network', 'optimizer', 'lr_scheduler', - 'train_data_loader', 'val_data_loader', 'run_summary'} + # Removing 'imputer', 'encoder', 'scaler', these will be + # TODO: added back after a PR fixing preprocessing + expected_keys = {'tabular_transformer', 'preprocess_transforms', 'network', + 'optimizer', 'lr_scheduler', 'train_data_loader', + 'val_data_loader', 'run_summary', 'feature_preprocessor'} assert expected_keys.issubset(set(transformed_fit_dictionary_tabular.keys())) # Then we need to have transformations being created. @@ -264,8 +270,8 @@ def test_error_search_space_updates(self, fit_dictionary_tabular, error_search_s exclude={'trainer': ['AdversarialTrainer']}) except Exception as e: assert isinstance(e, ValueError) - assert re.match(r'Unknown hyperparameter for component .*?\. Expected update ' - r'hyperparameter to be in \[.*?\] got .+', e.args[0]) + assert re.match(r'Unknown hyperparameter for .*?\. Expected update ' + r'hyperparameter to be in \[.*?\], but got .+', e.args[0]) def test_set_range_search_space_updates(self, fit_dictionary_tabular): dataset_properties = {'numerical_columns': [1], 'categorical_columns': [2], From 02e97a16dd55e19d8b27cc10e52716b15cf6226e Mon Sep 17 00:00:00 2001 From: Ravin Kohli <13005107+ravinkohli@users.noreply.github.com> Date: Fri, 10 Dec 2021 13:02:43 +0100 Subject: [PATCH 38/50] [FIX] Tests after rebase of `reg_cocktails` (#359) * update requirements * update requirements * resolve remaining conflicts and fix flake and mypy * Fix remaining tests and examples * fix failing checks * fix flake --- autoPyTorch/api/base_task.py | 75 +-- autoPyTorch/api/tabular_classification.py | 10 +- autoPyTorch/api/tabular_regression.py | 8 +- autoPyTorch/data/base_target_validator.py | 1 - autoPyTorch/data/tabular_feature_validator.py | 3 +- autoPyTorch/data/tabular_target_validator.py | 2 +- autoPyTorch/evaluation/fit_evaluator.py | 2 +- autoPyTorch/optimizer/smbo.py | 2 +- .../setup/network_backbone/utils.py | 47 +- .../setup/network_embedding/__init__.py | 101 +-- .../setup/network_head/fully_connected.py | 2 +- .../components/setup/network_head/no_head.py | 9 +- .../training/data_loader/base_data_loader.py | 3 +- .../training/trainer/AdversarialTrainer.py | 11 +- .../training/trainer/GridCutMixTrainer.py | 3 +- .../training/trainer/GridCutOutTrainer.py | 3 +- .../training/trainer/MixUpTrainer.py | 5 +- .../training/trainer/RowCutMixTrainer.py | 5 +- .../training/trainer/RowCutOutTrainer.py | 5 +- .../training/trainer/StandardTrainer.py | 8 +- .../components/training/trainer/__init__.py | 10 +- .../training/trainer/base_trainer.py | 8 +- .../training/trainer/cutout_utils.py | 5 +- .../training/trainer/mixup_utils.py | 5 +- .../components/training/trainer/utils.py | 4 +- .../pipeline/tabular_classification.py | 13 - autoPyTorch/utils/backend.py | 575 ------------------ .../example_custom_configuration_space.py | 141 ----- .../example_custom_configuration_space.py | 158 ++--- .../example_posthoc_ensemble_fit.py | 0 requirements.txt | 2 +- test/test_api/test_api.py | 1 - test/test_data/test_feature_validator.py | 21 +- test/test_evaluation/test_fit_evaluator.py | 7 +- .../components/setup/test_setup.py | 23 +- .../components/training/test_training.py | 14 +- .../test_tabular_classification.py | 10 +- test/test_pipeline/test_tabular_regression.py | 12 +- 38 files changed, 290 insertions(+), 1024 deletions(-) delete mode 100644 autoPyTorch/utils/backend.py delete mode 100644 examples/40_advanced/40_advanced/example_custom_configuration_space.py rename examples/40_advanced/{40_advanced => }/example_posthoc_ensemble_fit.py (100%) diff --git a/autoPyTorch/api/base_task.py b/autoPyTorch/api/base_task.py index ad3696101..37784c6f2 100644 --- a/autoPyTorch/api/base_task.py +++ b/autoPyTorch/api/base_task.py @@ -954,18 +954,15 @@ def run_traditional_ml( learning algorithm runs over the time limit. """ assert self._logger is not None # for mypy compliancy - if STRING_TO_TASK_TYPES[self.task_type] in REGRESSION_TASKS: - self._logger.warning("Traditional Pipeline is not enabled for regression. Skipping...") - else: - traditional_task_name = 'runTraditional' - self._stopwatch.start_task(traditional_task_name) - elapsed_time = self._stopwatch.wall_elapsed(current_task_name) - time_for_traditional = int(runtime_limit - elapsed_time) - self._do_traditional_prediction( - func_eval_time_limit_secs=func_eval_time_limit_secs, - time_left=time_for_traditional, - ) - self._stopwatch.stop_task(traditional_task_name) + traditional_task_name = 'runTraditional' + self._stopwatch.start_task(traditional_task_name) + elapsed_time = self._stopwatch.wall_elapsed(current_task_name) + time_for_traditional = int(runtime_limit - elapsed_time) + self._do_traditional_prediction( + func_eval_time_limit_secs=func_eval_time_limit_secs, + time_left=time_for_traditional, + ) + self._stopwatch.stop_task(traditional_task_name) def _search( self, @@ -1347,22 +1344,7 @@ def _search( self._logger.info("Starting Shutdown") if proc_ensemble is not None: - self._results_manager.ensemble_performance_history = list(proc_ensemble.history) - - if len(proc_ensemble.futures) > 0: - # Also add ensemble runs that did not finish within smac time - # and add them into the ensemble history - self._logger.info("Ensemble script still running, waiting for it to finish.") - result = proc_ensemble.futures.pop().result() - if result: - ensemble_history, _, _, _ = result - self._results_manager.ensemble_performance_history.extend(ensemble_history) - self._logger.info("Ensemble script finished, continue shutdown.") - - # save the ensemble performance history file - if len(self.ensemble_performance_history) > 0: - pd.DataFrame(self.ensemble_performance_history).to_json( - os.path.join(self._backend.internals_directory, 'ensemble_history.json')) + self._collect_results_ensemble(proc_ensemble) if load_models: self._logger.info("Loading models...") @@ -1641,7 +1623,7 @@ def fit_pipeline( exclude=self.exclude_components, search_space_updates=self.search_space_updates) dataset_properties = dataset.get_dataset_properties(dataset_requirements) - self._backend.replace_datamanager(dataset) + self._backend.save_datamanager(dataset) if self._logger is None: self._logger = self._get_logger(dataset.dataset_name) @@ -1832,7 +1814,7 @@ def fit_ensemble( ensemble_fit_task_name = 'EnsembleFit' self._stopwatch.start_task(ensemble_fit_task_name) if enable_traditional_pipeline: - if func_eval_time_limit_secs is None or func_eval_time_limit_secs > time_for_task: + if func_eval_time_limit_secs > time_for_task: self._logger.warning( 'Time limit for a single run is higher than total time ' 'limit. Capping the limit for a single run to the total ' @@ -1873,12 +1855,8 @@ def fit_ensemble( ) manager.build_ensemble(self._dask_client) - future = manager.futures.pop() - result = future.result() - if result is None: - raise ValueError("Errors occurred while building the ensemble - please" - " check the log file and command line output for error messages.") - self.ensemble_performance_history, _, _, _ = result + if manager is not None: + self._collect_results_ensemble(manager) if load_models: self._load_models() @@ -1956,6 +1934,31 @@ def _init_ensemble_builder( return proc_ensemble + def _collect_results_ensemble( + self, + manager: EnsembleBuilderManager + ) -> None: + + if self._logger is None: + raise ValueError("logger should be initialized to fit ensemble") + + self._results_manager.ensemble_performance_history = list(manager.history) + + if len(manager.futures) > 0: + # Also add ensemble runs that did not finish within smac time + # and add them into the ensemble history + self._logger.info("Ensemble script still running, waiting for it to finish.") + result = manager.futures.pop().result() + if result: + ensemble_history, _, _, _ = result + self._results_manager.ensemble_performance_history.extend(ensemble_history) + self._logger.info("Ensemble script finished, continue shutdown.") + + # save the ensemble performance history file + if len(self.ensemble_performance_history) > 0: + pd.DataFrame(self.ensemble_performance_history).to_json( + os.path.join(self._backend.internals_directory, 'ensemble_history.json')) + def predict( self, X_test: np.ndarray, diff --git a/autoPyTorch/api/tabular_classification.py b/autoPyTorch/api/tabular_classification.py index ed920960b..fdb1cee09 100644 --- a/autoPyTorch/api/tabular_classification.py +++ b/autoPyTorch/api/tabular_classification.py @@ -18,6 +18,7 @@ from autoPyTorch.datasets.base_dataset import BaseDatasetPropertiesType from autoPyTorch.datasets.resampling_strategy import ( HoldoutValTypes, + CrossValTypes, ResamplingStrategies, ) from autoPyTorch.datasets.tabular_dataset import TabularDataset @@ -449,6 +450,7 @@ def search( if self.dataset is None: raise ValueError("`dataset` in {} must be initialized, but got None".format(self.__class__.__name__)) + return self._search( dataset=self.dataset, optimize_metric=optimize_metric, @@ -488,23 +490,23 @@ def predict( raise ValueError("predict() is only supported after calling search. Kindly call first " "the estimator search() method.") - X_test = self.input_validator.feature_validator.transform(X_test) + X_test = self.InputValidator.feature_validator.transform(X_test) predicted_probabilities = super().predict(X_test, batch_size=batch_size, n_jobs=n_jobs) - if self.input_validator.target_validator.is_single_column_target(): + if self.InputValidator.target_validator.is_single_column_target(): predicted_indexes = np.argmax(predicted_probabilities, axis=1) else: predicted_indexes = (predicted_probabilities > 0.5).astype(int) # Allow to predict in the original domain -- that is, the user is not interested # in our encoded values - return self.input_validator.target_validator.inverse_transform(predicted_indexes) + return self.InputValidator.target_validator.inverse_transform(predicted_indexes) def predict_proba(self, X_test: Union[np.ndarray, pd.DataFrame, List], batch_size: Optional[int] = None, n_jobs: int = 1) -> np.ndarray: - if self.input_validator is None or not self.input_validator._is_fitted: + if self.InputValidator is None or not self.InputValidator._is_fitted: raise ValueError("predict() is only supported after calling search. Kindly call first " "the estimator search() method.") X_test = self.input_validator.feature_validator.transform(X_test) diff --git a/autoPyTorch/api/tabular_regression.py b/autoPyTorch/api/tabular_regression.py index d14e6891c..59b788266 100644 --- a/autoPyTorch/api/tabular_regression.py +++ b/autoPyTorch/api/tabular_regression.py @@ -18,6 +18,7 @@ from autoPyTorch.datasets.base_dataset import BaseDatasetPropertiesType from autoPyTorch.datasets.resampling_strategy import ( HoldoutValTypes, + CrossValTypes, ResamplingStrategies, ) from autoPyTorch.datasets.tabular_dataset import TabularDataset @@ -449,6 +450,7 @@ def search( if self.dataset is None: raise ValueError("`dataset` in {} must be initialized, but got None".format(self.__class__.__name__)) + return self._search( dataset=self.dataset, optimize_metric=optimize_metric, @@ -474,14 +476,14 @@ def predict( batch_size: Optional[int] = None, n_jobs: int = 1 ) -> np.ndarray: - if self.input_validator is None or not self.input_validator._is_fitted: + if self.InputValidator is None or not self.InputValidator._is_fitted: raise ValueError("predict() is only supported after calling search. Kindly call first " "the estimator search() method.") - X_test = self.input_validator.feature_validator.transform(X_test) + X_test = self.InputValidator.feature_validator.transform(X_test) predicted_values = super().predict(X_test, batch_size=batch_size, n_jobs=n_jobs) # Allow to predict in the original domain -- that is, the user is not interested # in our encoded values - return self.input_validator.target_validator.inverse_transform(predicted_values) + return self.InputValidator.target_validator.inverse_transform(predicted_values) diff --git a/autoPyTorch/data/base_target_validator.py b/autoPyTorch/data/base_target_validator.py index 5c209b9a5..84d0576c0 100644 --- a/autoPyTorch/data/base_target_validator.py +++ b/autoPyTorch/data/base_target_validator.py @@ -86,7 +86,6 @@ def fit( np.shape(y_test) )) if isinstance(y_train, pd.DataFrame): - y_train = cast(pd.DataFrame, y_train) y_test = cast(pd.DataFrame, y_test) if y_train.columns.tolist() != y_test.columns.tolist(): raise ValueError( diff --git a/autoPyTorch/data/tabular_feature_validator.py b/autoPyTorch/data/tabular_feature_validator.py index ad305c588..f1170f1a2 100644 --- a/autoPyTorch/data/tabular_feature_validator.py +++ b/autoPyTorch/data/tabular_feature_validator.py @@ -2,6 +2,7 @@ from logging import Logger from typing import Dict, List, Optional, Tuple, Union, cast + import numpy as np import pandas as pd @@ -275,7 +276,7 @@ def transform( if isinstance(X, np.ndarray): X = self.numpy_to_pandas(X) - if hasattr(X, "iloc") and not issparse(X): + if ispandas(X) and not issparse(X): X = cast(pd.DataFrame, X) # Check the data here so we catch problems on new test data diff --git a/autoPyTorch/data/tabular_target_validator.py b/autoPyTorch/data/tabular_target_validator.py index 3f1aa2f96..8f0c765f6 100644 --- a/autoPyTorch/data/tabular_target_validator.py +++ b/autoPyTorch/data/tabular_target_validator.py @@ -1,4 +1,4 @@ -from typing import List, Optional, Union, cast +from typing import List, Optional, cast import numpy as np import numpy.ma as ma diff --git a/autoPyTorch/evaluation/fit_evaluator.py b/autoPyTorch/evaluation/fit_evaluator.py index 281913003..f171cc18c 100644 --- a/autoPyTorch/evaluation/fit_evaluator.py +++ b/autoPyTorch/evaluation/fit_evaluator.py @@ -10,13 +10,13 @@ from smac.tae import StatusType +from autoPyTorch.automl_common.common.utils.backend import Backend from autoPyTorch.datasets.resampling_strategy import NoResamplingStrategyTypes from autoPyTorch.evaluation.abstract_evaluator import ( AbstractEvaluator, fit_and_suppress_warnings ) from autoPyTorch.pipeline.components.training.metrics.base import autoPyTorchMetric -from autoPyTorch.utils.backend import Backend from autoPyTorch.utils.common import subsampler from autoPyTorch.utils.hyperparameter_search_space_update import HyperparameterSearchSpaceUpdates diff --git a/autoPyTorch/optimizer/smbo.py b/autoPyTorch/optimizer/smbo.py index 53eae4696..fefa5cc12 100644 --- a/autoPyTorch/optimizer/smbo.py +++ b/autoPyTorch/optimizer/smbo.py @@ -120,7 +120,7 @@ def __init__(self, resampling_strategy_args: Optional[Dict[str, Any]] = None, include: Optional[Dict[str, Any]] = None, exclude: Optional[Dict[str, Any]] = None, - disable_file_output: List = [], + disable_file_output: Union[bool, List[str]] = False, smac_scenario_args: Optional[Dict[str, Any]] = None, get_smac_object_callback: Optional[Callable] = None, all_supported_metrics: bool = True, diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/utils.py b/autoPyTorch/pipeline/components/setup/network_backbone/utils.py index 80f8e4dc0..05e39fd09 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/utils.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/utils.py @@ -92,11 +92,7 @@ class ShakeDropFunction(Function): Github URL: https://github.com/owruby/shake-drop_pytorch/blob/master/models/shakedrop.py """ @staticmethod -<<<<<<< HEAD def forward(ctx: Any, -======= - def forward(ctx: typing.Any, ->>>>>>> Bug fixes (#249) x: torch.Tensor, alpha: torch.Tensor, beta: torch.Tensor, @@ -123,31 +119,20 @@ def backward(ctx: Any, shake_drop = ShakeDropFunction.apply -<<<<<<< HEAD -def shake_get_alpha_beta(is_training: bool, is_cuda: bool - ) -> Tuple[torch.Tensor, torch.Tensor]: - """ - The methods used in this function have been introduced in 'ShakeShake Regularisation' - Currently, this function supports `shake-shake`. -======= def shake_get_alpha_beta( is_training: bool, is_cuda: bool, method: str -) -> typing.Tuple[torch.Tensor, torch.Tensor]: +) -> Tuple[torch.Tensor, torch.Tensor]: """ The methods used in this function have been introduced in 'ShakeShake Regularisation' Each method name is available in the referred paper. Currently, this function supports `even-even`, `shake-even`, `shake-shake` and `M3`. ->>>>>>> Bug fixes (#249) Args: is_training (bool): Whether the computation for the training is_cuda (bool): Whether the tensor is on CUDA -<<<<<<< HEAD -======= method (str): The shake method either `even-even`, `shake-even`, `shake-shake` or `M3` ->>>>>>> Bug fixes (#249) Returns: alpha, beta (Tuple[float, float]): @@ -159,14 +144,8 @@ def shake_get_alpha_beta( Author: Xavier Gastaldi URL: https://arxiv.org/abs/1705.07485 -<<<<<<< HEAD - Note: - The names have been taken from the paper as well. - Currently, this function supports `shake-shake`. -======= The names have been taken from the paper as well. Currently, this function supports `even-even`, `shake-even`, `shake-shake` and `M3`. ->>>>>>> Bug fixes (#249) """ if not is_training: result = (torch.FloatTensor([0.5]), torch.FloatTensor([0.5])) @@ -196,27 +175,15 @@ def shake_get_alpha_beta( def shake_drop_get_bl( -<<<<<<< HEAD - block_index: int, - min_prob_no_shake: float, - num_blocks: int, - is_training: bool, - is_cuda: bool -======= block_index: int, min_prob_no_shake: float, num_blocks: int, is_training: bool, is_cuda: bool ->>>>>>> Bug fixes (#249) ) -> torch.Tensor: """ The sampling of Bernoulli random variable based on Eq. (4) in the paper -<<<<<<< HEAD - -======= ->>>>>>> Bug fixes (#249) Args: block_index (int): The index of the block from the input layer min_prob_no_shake (float): The initial shake probability @@ -226,28 +193,16 @@ def shake_drop_get_bl( Returns: bl (torch.Tensor): a Bernoulli random variable in {0, 1} -<<<<<<< HEAD - -======= ->>>>>>> Bug fixes (#249) Reference: ShakeDrop Regularization for Deep Residual Learning Yoshihiro Yamada et. al. (2020) paper: https://arxiv.org/pdf/1802.02375.pdf implementation: https://github.com/imenurok/ShakeDrop """ -<<<<<<< HEAD - - pl = 1 - ((block_index + 1) / num_blocks) * (1 - min_prob_no_shake) - - if is_training: - # Move to torch.rand(1) for reproducibility -======= pl = 1 - ((block_index + 1) / num_blocks) * (1 - min_prob_no_shake) if is_training: # Move to torch.randn(1) for reproducibility ->>>>>>> Bug fixes (#249) bl = torch.as_tensor(1.0) if torch.rand(1) <= pl else torch.as_tensor(0.0) else: bl = torch.as_tensor(pl) diff --git a/autoPyTorch/pipeline/components/setup/network_embedding/__init__.py b/autoPyTorch/pipeline/components/setup/network_embedding/__init__.py index 452e74cc1..86b8b899d 100644 --- a/autoPyTorch/pipeline/components/setup/network_embedding/__init__.py +++ b/autoPyTorch/pipeline/components/setup/network_embedding/__init__.py @@ -146,62 +146,71 @@ def get_hyperparameter_search_space( if default is None: defaults = [ 'NoEmbedding', - 'LearnedEntityEmbedding', + # 'LearnedEntityEmbedding', ] for default_ in defaults: if default_ in available_embedding: default = default_ break - categorical_columns = dataset_properties['categorical_columns'] \ - if isinstance(dataset_properties['categorical_columns'], List) else [] - - updates = self._get_search_space_updates() - if '__choice__' in updates.keys(): - choice_hyperparameter = updates['__choice__'] - if not set(choice_hyperparameter.value_range).issubset(available_embedding): - raise ValueError("Expected given update for {} to have " - "choices in {} got {}".format(self.__class__.__name__, - available_embedding, - choice_hyperparameter.value_range)) - if len(categorical_columns) == 0: - assert len(choice_hyperparameter.value_range) == 1 - if 'NoEmbedding' not in choice_hyperparameter.value_range: - raise ValueError("Provided {} in choices, however, the dataset " - "is incompatible with it".format(choice_hyperparameter.value_range)) - embedding = CSH.CategoricalHyperparameter('__choice__', - choice_hyperparameter.value_range, - default_value=choice_hyperparameter.default_value) - else: - - if len(categorical_columns) == 0: - default = 'NoEmbedding' - if include is not None and default not in include: - raise ValueError("Provided {} in include, however, the dataset " - "is incompatible with it".format(include)) - embedding = CSH.CategoricalHyperparameter('__choice__', - ['NoEmbedding'], - default_value=default) - else: - embedding = CSH.CategoricalHyperparameter('__choice__', - list(available_embedding.keys()), - default_value=default) - + # Restrict embedding to NoEmbedding until preprocessing is fixed + embedding = CSH.CategoricalHyperparameter('__choice__', + ['NoEmbedding'], + default_value=default) cs.add_hyperparameter(embedding) - for name in embedding.choices: - updates = self._get_search_space_updates(prefix=name) - config_space = available_embedding[name].get_hyperparameter_search_space(dataset_properties, # type: ignore - **updates) - parent_hyperparameter = {'parent': embedding, 'value': name} - cs.add_configuration_space( - name, - config_space, - parent_hyperparameter=parent_hyperparameter - ) - self.configuration_space_ = cs self.dataset_properties_ = dataset_properties return cs + # categorical_columns = dataset_properties['categorical_columns'] \ + # if isinstance(dataset_properties['categorical_columns'], List) else [] + + # updates = self._get_search_space_updates() + # if '__choice__' in updates.keys(): + # choice_hyperparameter = updates['__choice__'] + # if not set(choice_hyperparameter.value_range).issubset(available_embedding): + # raise ValueError("Expected given update for {} to have " + # "choices in {} got {}".format(self.__class__.__name__, + # available_embedding, + # choice_hyperparameter.value_range)) + # if len(categorical_columns) == 0: + # assert len(choice_hyperparameter.value_range) == 1 + # if 'NoEmbedding' not in choice_hyperparameter.value_range: + # raise ValueError("Provided {} in choices, however, the dataset " + # "is incompatible with it".format(choice_hyperparameter.value_range)) + # embedding = CSH.CategoricalHyperparameter('__choice__', + # choice_hyperparameter.value_range, + # default_value=choice_hyperparameter.default_value) + # else: + + # if len(categorical_columns) == 0: + # default = 'NoEmbedding' + # if include is not None and default not in include: + # raise ValueError("Provided {} in include, however, the dataset " + # "is incompatible with it".format(include)) + # embedding = CSH.CategoricalHyperparameter('__choice__', + # ['NoEmbedding'], + # default_value=default) + # else: + # embedding = CSH.CategoricalHyperparameter('__choice__', + # list(available_embedding.keys()), + # default_value=default) + + # cs.add_hyperparameter(embedding) + # for name in embedding.choices: + # updates = self._get_search_space_updates(prefix=name) + # config_space = available_embedding[name].get_hyperparameter_search_space( + # dataset_properties, # type: ignore + # **updates) + # parent_hyperparameter = {'parent': embedding, 'value': name} + # cs.add_configuration_space( + # name, + # config_space, + # parent_hyperparameter=parent_hyperparameter + # ) + + # self.configuration_space_ = cs + # self.dataset_properties_ = dataset_properties + # return cs def transform(self, X: Dict[str, Any]) -> Dict[str, Any]: assert self.choice is not None, "Cannot call transform before the object is initialized" diff --git a/autoPyTorch/pipeline/components/setup/network_head/fully_connected.py b/autoPyTorch/pipeline/components/setup/network_head/fully_connected.py index 3c01f75da..8f1d75040 100644 --- a/autoPyTorch/pipeline/components/setup/network_head/fully_connected.py +++ b/autoPyTorch/pipeline/components/setup/network_head/fully_connected.py @@ -82,7 +82,7 @@ def get_hyperparameter_search_space( log=units_layer.log, ) num_units_hp = get_hyperparameter(num_units_search_space, UniformIntegerHyperparameter) - + cs.add_hyperparameter(num_units_hp) if i >= min_num_layers and not num_layers_is_constant: # In the case of a constant, the max and min number of layers are the same. # So no condition is needed. If it is not a constant but a hyperparameter, diff --git a/autoPyTorch/pipeline/components/setup/network_head/no_head.py b/autoPyTorch/pipeline/components/setup/network_head/no_head.py index 0e711f06c..e95d25ffb 100644 --- a/autoPyTorch/pipeline/components/setup/network_head/no_head.py +++ b/autoPyTorch/pipeline/components/setup/network_head/no_head.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple, Union +from typing import Dict, Optional, Tuple, Union from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import CategoricalHyperparameter @@ -7,6 +7,7 @@ from torch import nn +from autoPyTorch.datasets.base_dataset import BaseDatasetPropertiesType from autoPyTorch.pipeline.components.setup.network_head.base_network_head import NetworkHeadComponent from autoPyTorch.pipeline.components.setup.network_head.utils import _activations from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter @@ -28,7 +29,9 @@ def build_head(self, input_shape: Tuple[int, ...], output_shape: Tuple[int, ...] return nn.Sequential(*layers) @staticmethod - def get_properties(dataset_properties: Optional[Dict[str, Any]] = None) -> Dict[str, Union[str, bool]]: + def get_properties( + dataset_properties: Optional[Dict[str, BaseDatasetPropertiesType]] = None + ) -> Dict[str, Union[str, bool]]: return { 'shortname': 'NoHead', 'name': 'NoHead', @@ -39,7 +42,7 @@ def get_properties(dataset_properties: Optional[Dict[str, Any]] = None) -> Dict[ @staticmethod def get_hyperparameter_search_space( - dataset_properties: Optional[Dict[str, str]] = None, + dataset_properties: Optional[Dict[str, BaseDatasetPropertiesType]] = None, activation: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="activation", value_range=tuple(_activations.keys()), default_value=list(_activations.keys())[0]), diff --git a/autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py b/autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py index c601e4a3d..d99ba055c 100644 --- a/autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py +++ b/autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py @@ -115,7 +115,7 @@ def fit(self, X: Dict[str, Any], y: Any = None) -> torch.utils.data.DataLoader: shuffle=True, num_workers=X.get('num_workers', 0), pin_memory=X.get('pin_memory', True), - drop_last=X.get('drop_last', False), + drop_last=X.get('drop_last', True), collate_fn=custom_collate_fn, ) @@ -149,7 +149,6 @@ def get_loader(self, X: np.ndarray, y: Optional[np.ndarray] = None, batch_size: train_tensors=(X, y), seed=self.random_state.get_state()[1][0], # This dataset is used for loading test data in a batched format - seed=self.random_state.get_state()[1][0], shuffle=False, train_transforms=self.test_transform, val_transforms=self.test_transform, diff --git a/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py b/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py index 0fefd9525..67ae71188 100644 --- a/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py @@ -15,6 +15,7 @@ from autoPyTorch.constants import CLASSIFICATION_TASKS, STRING_TO_TASK_TYPES +from autoPyTorch.datasets.base_dataset import BaseDatasetPropertiesType from autoPyTorch.pipeline.components.training.trainer.base_trainer import BaseTrainerComponent from autoPyTorch.pipeline.components.training.trainer.utils import Lookahead from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter @@ -91,7 +92,7 @@ def train_step(self, data: np.ndarray, targets: np.ndarray) -> Tuple[float, torc """ # prepare data = data.float().to(self.device) - targets = targets.long().to(self.device) + targets = self.cast_targets(targets) data, criterion_kwargs = self.data_preparation(data, targets) original_data = data[0] @@ -131,7 +132,7 @@ def fgsm_attack( """ data_copy = deepcopy(data) data_copy = data_copy.float().to(self.device) - targets = targets.long().to(self.device) + targets = self.cast_targets(targets) data_copy = torch.autograd.Variable(data_copy) data_copy.requires_grad = True @@ -146,7 +147,7 @@ def fgsm_attack( return adv_data @staticmethod - def get_properties(dataset_properties: Optional[Dict[str, Any]] = None + def get_properties(dataset_properties: Optional[Dict[str, BaseDatasetPropertiesType]] = None ) -> Dict[str, Union[str, bool]]: return { @@ -159,7 +160,7 @@ def get_properties(dataset_properties: Optional[Dict[str, Any]] = None @staticmethod def get_hyperparameter_search_space( - dataset_properties: Optional[Dict] = None, + dataset_properties: Optional[Dict[str, BaseDatasetPropertiesType]] = None, weighted_loss: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="weighted_loss", value_range=(1, ), @@ -240,7 +241,7 @@ def get_hyperparameter_search_space( # remove the code below. Also update the method signature, so the weighted loss # is not a constant. if dataset_properties is not None: - if STRING_TO_TASK_TYPES[dataset_properties['task_type']] in CLASSIFICATION_TASKS: + if STRING_TO_TASK_TYPES[str(dataset_properties['task_type'])] in CLASSIFICATION_TASKS: add_hyperparameter(cs, weighted_loss, Constant) return cs diff --git a/autoPyTorch/pipeline/components/training/trainer/GridCutMixTrainer.py b/autoPyTorch/pipeline/components/training/trainer/GridCutMixTrainer.py index 24346042d..9bf22f3b8 100644 --- a/autoPyTorch/pipeline/components/training/trainer/GridCutMixTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/GridCutMixTrainer.py @@ -4,6 +4,7 @@ import torch +from autoPyTorch.datasets.base_dataset import BaseDatasetPropertiesType from autoPyTorch.pipeline.components.training.trainer.base_trainer import BaseTrainerComponent from autoPyTorch.pipeline.components.training.trainer.mixup_utils import MixUp @@ -67,7 +68,7 @@ def data_preparation(self, X: np.ndarray, y: np.ndarray, return X, {'y_a': y_a, 'y_b': y_b, 'lam': lam} @staticmethod - def get_properties(dataset_properties: typing.Optional[typing.Dict[str, typing.Any]] = None + def get_properties(dataset_properties: typing.Optional[typing.Dict[str, BaseDatasetPropertiesType]] = None ) -> typing.Dict[str, typing.Union[str, bool]]: return { 'shortname': 'GridCutMixTrainer', diff --git a/autoPyTorch/pipeline/components/training/trainer/GridCutOutTrainer.py b/autoPyTorch/pipeline/components/training/trainer/GridCutOutTrainer.py index 4d7f1099d..fb6389fb8 100644 --- a/autoPyTorch/pipeline/components/training/trainer/GridCutOutTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/GridCutOutTrainer.py @@ -2,6 +2,7 @@ import numpy as np +from autoPyTorch.datasets.base_dataset import BaseDatasetPropertiesType from autoPyTorch.pipeline.components.training.trainer.base_trainer import BaseTrainerComponent from autoPyTorch.pipeline.components.training.trainer.cutout_utils import CutOut @@ -52,7 +53,7 @@ def data_preparation(self, X: np.ndarray, y: np.ndarray, return X, {'y_a': y, 'y_b': y, 'lam': 1} @staticmethod - def get_properties(dataset_properties: typing.Optional[typing.Dict[str, typing.Any]] = None + def get_properties(dataset_properties: typing.Optional[typing.Dict[str, BaseDatasetPropertiesType]] = None ) -> typing.Dict[str, typing.Union[str, bool]]: return { 'shortname': 'GridCutOutTrainer', diff --git a/autoPyTorch/pipeline/components/training/trainer/MixUpTrainer.py b/autoPyTorch/pipeline/components/training/trainer/MixUpTrainer.py index 2bd015b46..1cd071ba6 100644 --- a/autoPyTorch/pipeline/components/training/trainer/MixUpTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/MixUpTrainer.py @@ -1,9 +1,10 @@ -from typing import Any, Dict, Optional, Tuple, Union +from typing import Dict, Optional, Tuple, Union import numpy as np import torch +from autoPyTorch.datasets.base_dataset import BaseDatasetPropertiesType from autoPyTorch.pipeline.components.training.trainer.base_trainer import BaseTrainerComponent from autoPyTorch.pipeline.components.training.trainer.mixup_utils import MixUp @@ -43,7 +44,7 @@ def data_preparation(self, X: np.ndarray, y: np.ndarray, return mixed_x, {'y_a': y_a, 'y_b': y_b, 'lam': lam} @staticmethod - def get_properties(dataset_properties: Optional[Dict[str, Any]] = None + def get_properties(dataset_properties: Optional[Dict[str, BaseDatasetPropertiesType]] = None ) -> Dict[str, Union[str, bool]]: return { 'shortname': 'MixUpTrainer', diff --git a/autoPyTorch/pipeline/components/training/trainer/RowCutMixTrainer.py b/autoPyTorch/pipeline/components/training/trainer/RowCutMixTrainer.py index 3f7866f3c..bb4ccdb9a 100644 --- a/autoPyTorch/pipeline/components/training/trainer/RowCutMixTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/RowCutMixTrainer.py @@ -1,9 +1,10 @@ -from typing import Any, Dict, Optional, Tuple, Union +from typing import Dict, Optional, Tuple, Union import numpy as np import torch +from autoPyTorch.datasets.base_dataset import BaseDatasetPropertiesType from autoPyTorch.pipeline.components.training.trainer.base_trainer import BaseTrainerComponent from autoPyTorch.pipeline.components.training.trainer.mixup_utils import MixUp @@ -56,7 +57,7 @@ def data_preparation(self, X: np.ndarray, y: np.ndarray, return X, {'y_a': y_a, 'y_b': y_b, 'lam': lam} @staticmethod - def get_properties(dataset_properties: Optional[Dict[str, Any]] = None + def get_properties(dataset_properties: Optional[Dict[str, BaseDatasetPropertiesType]] = None ) -> Dict[str, Union[str, bool]]: return { 'shortname': 'RowCutMixTrainer', diff --git a/autoPyTorch/pipeline/components/training/trainer/RowCutOutTrainer.py b/autoPyTorch/pipeline/components/training/trainer/RowCutOutTrainer.py index 4578082cb..7b679976e 100644 --- a/autoPyTorch/pipeline/components/training/trainer/RowCutOutTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/RowCutOutTrainer.py @@ -1,7 +1,8 @@ -from typing import Any, Dict, Optional, Tuple, Union +from typing import Dict, Optional, Tuple, Union import numpy as np +from autoPyTorch.datasets.base_dataset import BaseDatasetPropertiesType from autoPyTorch.pipeline.components.training.trainer.base_trainer import BaseTrainerComponent from autoPyTorch.pipeline.components.training.trainer.cutout_utils import CutOut @@ -53,7 +54,7 @@ def data_preparation(self, X: np.ndarray, y: np.ndarray, return X, {'y_a': y_a, 'y_b': y_b, 'lam': lam} @staticmethod - def get_properties(dataset_properties: Optional[Dict[str, Any]] = None + def get_properties(dataset_properties: Optional[Dict[str, BaseDatasetPropertiesType]] = None ) -> Dict[str, Union[str, bool]]: return { 'shortname': 'RowCutOutTrainer', diff --git a/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py b/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py index fc5cc3e3a..c9202945c 100644 --- a/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py @@ -1,13 +1,9 @@ -from typing import Callable, Dict, Optional, Tuple, Union - -from ConfigSpace.configuration_space import ConfigurationSpace -from ConfigSpace.hyperparameters import CategoricalHyperparameter +from typing import Any, Callable, Dict, Optional, Tuple, Union import numpy as np import torch -from autoPyTorch.constants import CLASSIFICATION_TASKS, STRING_TO_TASK_TYPES from autoPyTorch.datasets.base_dataset import BaseDatasetPropertiesType from autoPyTorch.pipeline.components.training.trainer.base_trainer import BaseTrainerComponent @@ -60,7 +56,7 @@ def criterion_preparation(self, y_a: torch.Tensor, y_b: torch.Tensor = None, lam return lambda criterion, pred: criterion(pred, y_a) @staticmethod - def get_properties(dataset_properties: Optional[Dict[str, Any]] = None + def get_properties(dataset_properties: Optional[Dict[str, BaseDatasetPropertiesType]] = None ) -> Dict[str, Union[str, bool]]: return { 'shortname': 'StandardTrainer', diff --git a/autoPyTorch/pipeline/components/training/trainer/__init__.py b/autoPyTorch/pipeline/components/training/trainer/__init__.py index b740b8b53..144f703f7 100755 --- a/autoPyTorch/pipeline/components/training/trainer/__init__.py +++ b/autoPyTorch/pipeline/components/training/trainer/__init__.py @@ -129,7 +129,7 @@ def get_available_components( # Allow training schemes exclusive for some task types entry = available_comp[name] - task_type = dataset_properties['task_type'] + task_type = str(dataset_properties['task_type']) properties = entry.get_properties() if 'tabular' in task_type and not properties['handles_tabular']: continue @@ -283,9 +283,14 @@ def fit(self, X: Dict[str, Any], y: Any = None, **kwargs: Any) -> autoPyTorchCom **kwargs ) + # Comply with mypy + # Notice that choice here stands for the component choice framework, + # where we dynamically build the configuration space by selecting the available + # component choices. In this case, is what trainer choices are available + assert self.choice is not None + # Add snapshots to base network to enable # predicting with snapshot ensemble - self.choice: autoPyTorchComponent = cast(autoPyTorchComponent, self.choice) if self.choice.use_snapshot_ensemble: X['network_snapshots'].extend(self.choice.model_snapshots) return self.choice @@ -526,7 +531,6 @@ def early_stop_handler(self, X: Dict[str, Any]) -> bool: X (Dict[str, Any]): Dictionary with fitted parameters. It is a message passing mechanism, in which during a transform, a components adds relevant information so that further stages can be properly fitted - Returns: bool: If true, training should be stopped """ diff --git a/autoPyTorch/pipeline/components/training/trainer/base_trainer.py b/autoPyTorch/pipeline/components/training/trainer/base_trainer.py index 07c3877e2..7daca56d0 100644 --- a/autoPyTorch/pipeline/components/training/trainer/base_trainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/base_trainer.py @@ -20,9 +20,9 @@ from torch.optim.lr_scheduler import _LRScheduler from torch.utils.tensorboard.writer import SummaryWriter -from autoPyTorch.constants import FORECASTING_TASKS, REGRESSION_TASKS +from autoPyTorch.constants import CLASSIFICATION_TASKS, FORECASTING_TASKS, REGRESSION_TASKS, STRING_TO_TASK_TYPES +from autoPyTorch.datasets.base_dataset import BaseDatasetPropertiesType from autoPyTorch.pipeline.components.setup.lr_scheduler.constants import StepIntervalUnit -from autoPyTorch.constants import REGRESSION_TASKS, CLASSIFICATION_TASKS, STRING_TO_TASK_TYPES from autoPyTorch.pipeline.components.training.base_training import autoPyTorchTrainingComponent from autoPyTorch.pipeline.components.training.metrics.metrics import ( CLASSIFICATION_METRICS, @@ -602,7 +602,7 @@ def criterion_preparation(self, y_a: torch.Tensor, y_b: torch.Tensor = None, lam @staticmethod def get_hyperparameter_search_space( - dataset_properties: Optional[Dict] = None, + dataset_properties: Optional[Dict[str, BaseDatasetPropertiesType]] = None, weighted_loss: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="weighted_loss", value_range=(1, ), @@ -672,7 +672,7 @@ def get_hyperparameter_search_space( # remove the code below. Also update the method signature, so the weighted loss # is not a constant. if dataset_properties is not None: - if STRING_TO_TASK_TYPES[dataset_properties['task_type']] in CLASSIFICATION_TASKS: + if STRING_TO_TASK_TYPES[str(dataset_properties['task_type'])] in CLASSIFICATION_TASKS: add_hyperparameter(cs, weighted_loss, Constant) return cs diff --git a/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py b/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py index 4feedf5cb..a181fe530 100644 --- a/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py +++ b/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py @@ -13,6 +13,7 @@ from sklearn.utils import check_random_state from autoPyTorch.constants import CLASSIFICATION_TASKS, STRING_TO_TASK_TYPES +from autoPyTorch.datasets.base_dataset import BaseDatasetPropertiesType from autoPyTorch.pipeline.components.training.trainer.utils import Lookahead from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter @@ -60,7 +61,7 @@ def criterion_preparation(self, y_a: np.ndarray, y_b: np.ndarray = None, lam: fl @staticmethod def get_hyperparameter_search_space( - dataset_properties: Optional[Dict] = None, + dataset_properties: Optional[Dict[str, BaseDatasetPropertiesType]] = None, weighted_loss: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="weighted_loss", value_range=(1, ), @@ -146,7 +147,7 @@ def get_hyperparameter_search_space( # remove the code below. Also update the method signature, so the weighted loss # is not a constant. if dataset_properties is not None: - if STRING_TO_TASK_TYPES[dataset_properties['task_type']] in CLASSIFICATION_TASKS: + if STRING_TO_TASK_TYPES[str(dataset_properties['task_type'])] in CLASSIFICATION_TASKS: add_hyperparameter(cs, weighted_loss, Constant) return cs diff --git a/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py b/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py index e2ea25148..f9cd278a9 100644 --- a/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py +++ b/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py @@ -13,6 +13,7 @@ from sklearn.utils import check_random_state from autoPyTorch.constants import CLASSIFICATION_TASKS, STRING_TO_TASK_TYPES +from autoPyTorch.datasets.base_dataset import BaseDatasetPropertiesType from autoPyTorch.pipeline.components.training.trainer.utils import Lookahead from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter @@ -65,7 +66,7 @@ def criterion_preparation(self, y_a: np.ndarray, y_b: np.ndarray = None, lam: fl @staticmethod def get_hyperparameter_search_space( - dataset_properties: Optional[Dict] = None, + dataset_properties: Optional[Dict[str, BaseDatasetPropertiesType]] = None, weighted_loss: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="weighted_loss", value_range=(1, ), @@ -145,7 +146,7 @@ def get_hyperparameter_search_space( # remove the code below. Also update the method signature, so the weighted loss # is not a constant. if dataset_properties is not None: - if STRING_TO_TASK_TYPES[dataset_properties['task_type']] in CLASSIFICATION_TASKS: + if STRING_TO_TASK_TYPES[str(dataset_properties['task_type'])] in CLASSIFICATION_TASKS: add_hyperparameter(cs, weighted_loss, Constant) return cs diff --git a/autoPyTorch/pipeline/components/training/trainer/utils.py b/autoPyTorch/pipeline/components/training/trainer/utils.py index cdc22402f..ce16d5e3c 100644 --- a/autoPyTorch/pipeline/components/training/trainer/utils.py +++ b/autoPyTorch/pipeline/components/training/trainer/utils.py @@ -105,7 +105,7 @@ def get_la_step(self) -> int: return self._la_step def state_dict(self) -> Dict[str, Any]: - return self.optimizer.state_dict() + return self.optimizer.state_dict() # type: ignore[no-any-return] def load_state_dict(self, state_dict: Dict[str, Any]) -> None: self.optimizer.load_state_dict(state_dict) @@ -129,7 +129,7 @@ def _clear_and_load_backup(self) -> None: @property def param_groups(self) -> List[Dict]: - return self.optimizer.param_groups + return self.optimizer.param_groups # type: ignore[no-any-return] def step(self, closure: Optional[Callable] = None) -> torch.Tensor: """Performs a single Lookahead optimization step. diff --git a/autoPyTorch/pipeline/tabular_classification.py b/autoPyTorch/pipeline/tabular_classification.py index bc9616f58..720d0af64 100644 --- a/autoPyTorch/pipeline/tabular_classification.py +++ b/autoPyTorch/pipeline/tabular_classification.py @@ -142,23 +142,10 @@ def _predict_proba(self, X: np.ndarray) -> np.ndarray: # The final layer is always softmax now (`pred` already gives pseudo proba) return pred else: -<<<<<<< HEAD raise ValueError("Expected output_shape to be integer, got {}," "Tabular Classification only supports 'binary' and 'multiclass' outputs" "got {}".format(type(self.dataset_properties['output_shape']), self.dataset_properties['output_type'])) -======= - all_proba = [] - - for k in range(self.dataset_properties['output_shape']): - proba_k = pred[:, k, :self.dataset_properties['output_shape'][k]] - normalizer = proba_k.sum(axis=1)[:, np.newaxis] - normalizer[normalizer == 0.0] = 1.0 - proba_k /= normalizer - all_proba.append(proba_k) - - return np.array(all_proba) ->>>>>>> Bug fixes (#249) def predict_proba(self, X: np.ndarray, batch_size: Optional[int] = None) -> np.ndarray: """predict_proba. diff --git a/autoPyTorch/utils/backend.py b/autoPyTorch/utils/backend.py deleted file mode 100644 index 5348bd11c..000000000 --- a/autoPyTorch/utils/backend.py +++ /dev/null @@ -1,575 +0,0 @@ -import glob -import os -import pickle -import re -import shutil -import tempfile -import time -import uuid -import warnings -from typing import Dict, List, Optional, Tuple, Union - -import lockfile - -import numpy as np - -from autoPyTorch.datasets.base_dataset import BaseDataset -from autoPyTorch.ensemble.abstract_ensemble import AbstractEnsemble -from autoPyTorch.pipeline.base_pipeline import BasePipeline -from autoPyTorch.utils.logging_ import PicklableClientLogger, get_named_client_logger - -__all__ = [ - 'Backend' -] - - -def create( - temporary_directory: Optional[str], - output_directory: Optional[str], - delete_tmp_folder_after_terminate: bool = True, - delete_output_folder_after_terminate: bool = True, -) -> 'Backend': - """ - Creates a backend object that manages disk related transactions - - Args: - temporary_directory (str): where all temporal data is to be dumped - output_directory (str): where all predictions are to be output - delete_tmp_folder_after_terminate (bool): whether to delete the - temporal directory when then run completes - delete_output_folder_after_terminate (bool): whether to delete - the output directory when the run completes - - Returns: - Backend object - """ - context = BackendContext(temporary_directory, output_directory, - delete_tmp_folder_after_terminate, - delete_output_folder_after_terminate, - ) - backend = Backend(context) - - return backend - - -def get_randomized_directory_name(temporary_directory: Optional[str] = None) -> str: - uuid_str = str(uuid.uuid1(clock_seq=os.getpid())) - - temporary_directory = ( - temporary_directory - if temporary_directory - else os.path.join( - tempfile.gettempdir(), - "autoPyTorch_tmp_{}".format( - uuid_str, - ), - ) - ) - - return temporary_directory - - -class BackendContext(object): - - def __init__(self, - temporary_directory: Optional[str], - output_directory: Optional[str], - delete_tmp_folder_after_terminate: bool, - delete_output_folder_after_terminate: bool, - ): - - # Check that the names of tmp_dir and output_dir is not the same. - if temporary_directory == output_directory and temporary_directory is not None: - raise ValueError("The temporary and the output directory " - "must be different.") - - self.delete_tmp_folder_after_terminate = delete_tmp_folder_after_terminate - self.delete_output_folder_after_terminate = delete_output_folder_after_terminate - # attributes to check that directories were created by autoPyTorch - self._tmp_dir_created = False - self._output_dir_created = False - - self._temporary_directory = ( - get_randomized_directory_name( - temporary_directory=temporary_directory, - ) - ) - self._output_directory = output_directory - self.create_directories() - self._logger = None # type: Optional[PicklableClientLogger] - - @property - def output_directory(self) -> Optional[str]: - if self._output_directory is not None: - # make sure that tilde does not appear on the path. - return os.path.expanduser(os.path.expandvars(self._output_directory)) - else: - return None - - @property - def temporary_directory(self) -> str: - # make sure that tilde does not appear on the path. - return os.path.expanduser(os.path.expandvars(self._temporary_directory)) - - def create_directories(self) -> None: - # Exception is raised if self.temporary_directory already exists. - os.makedirs(self.temporary_directory) - self._tmp_dir_created = True - - # Exception is raised if self.output_directory already exists. - if self.output_directory is not None: - os.makedirs(self.output_directory) - self._output_dir_created = True - - def delete_directories(self, force: bool = True) -> None: - if self.output_directory and (self.delete_output_folder_after_terminate or force): - if self._output_dir_created is False: - raise ValueError("Failed to delete output dir: %s because autoPyTorch did not " - "create it. Please make sure that the specified output dir does " - "not exist when instantiating autoPyTorch." - % self.output_directory) - try: - shutil.rmtree(self.output_directory) - except Exception: - try: - if self._logger is not None: - self._logger.warning("Could not delete output dir: %s" % - self.output_directory) - else: - warnings.warn("Could not delete output dir: %s" % self.output_directory) - except Exception: - warnings.warn("Could not delete output dir: %s" % self.output_directory) - - if self.delete_tmp_folder_after_terminate or force: - if self._tmp_dir_created is False: - raise ValueError("Failed to delete tmp dir: % s because autoPyTorch did not " - "create it. Please make sure that the specified tmp dir does not " - "exist when instantiating autoPyTorch." - % self.temporary_directory) - try: - shutil.rmtree(self.temporary_directory) - except Exception: - try: - if self._logger is not None: - self._logger.warning( - "Could not delete tmp dir: %s" % self.temporary_directory) - else: - warnings.warn("Could not delete tmp dir: %s" % self.temporary_directory) - except Exception: - warnings.warn("Could not delete tmp dir: %s" % self.temporary_directory) - - -class Backend(object): - """Utility class to load and save all objects to be persisted. - These are: - * start time of auto-pytorch - * true targets of the ensemble - """ - - def __init__(self, context: BackendContext): - self._logger = None # type: Optional[PicklableClientLogger] - self.context = context - - # Track the number of configurations launched - # num_run == 1 means a dummy estimator run - self.active_num_run = 1 - - # Create the temporary directory if it does not yet exist - try: - os.makedirs(self.temporary_directory) - except Exception: - pass - # This does not have to exist or be specified - if self.output_directory is not None: - if not os.path.exists(self.output_directory): - raise ValueError("Output directory %s does not exist." % self.output_directory) - - self.internals_directory = os.path.join(self.temporary_directory, ".autoPyTorch") - self._make_internals_directory() - - def setup_logger(self, name: str, port: int) -> None: - self._logger = get_named_client_logger( - name=name, - port=port, - ) - self.context._logger = self._logger - return - - @property - def output_directory(self) -> Optional[str]: - return self.context.output_directory - - @property - def temporary_directory(self) -> str: - return self.context.temporary_directory - - def _make_internals_directory(self) -> None: - # TODO: make exist_ok a function argument - try: - os.makedirs(self.internals_directory, exist_ok=True) - except Exception as e: - if self._logger is not None: - self._logger.debug("_make_internals_directory: %s" % e) - try: - os.makedirs(self.get_runs_directory(), exist_ok=True) - except Exception as e: - if self._logger is not None: - self._logger.debug("_make_internals_directory: %s" % e) - - def _get_start_time_filename(self, seed: Union[str, int]) -> str: - if isinstance(seed, str): - seed = int(seed) - return os.path.join(self.internals_directory, "start_time_%d" % seed) - - def save_start_time(self, seed: str) -> str: - self._make_internals_directory() - start_time = time.time() - - filepath = self._get_start_time_filename(seed) - - if not isinstance(start_time, float): - raise ValueError("Start time must be a float, but is %s." % type(start_time)) - - if os.path.exists(filepath): - raise ValueError( - "{filepath} already exist. Different seeds should be provided for different jobs." - ) - - with tempfile.NamedTemporaryFile('w', dir=os.path.dirname(filepath), delete=False) as fh: - fh.write(str(start_time)) - tempname = fh.name - os.rename(tempname, filepath) - - return filepath - - def load_start_time(self, seed: int) -> float: - with open(self._get_start_time_filename(seed), 'r') as fh: - start_time = float(fh.read()) - return start_time - - def get_smac_output_directory(self) -> str: - return os.path.join(self.temporary_directory, 'smac3-output') - - def get_smac_output_directory_for_run(self, seed: int) -> str: - return os.path.join( - self.temporary_directory, - 'smac3-output', - 'run_%d' % seed - ) - - def _get_targets_ensemble_filename(self) -> str: - return os.path.join(self.internals_directory, - "true_targets_ensemble.npy") - - def save_targets_ensemble(self, targets: np.ndarray) -> str: - self._make_internals_directory() - if not isinstance(targets, np.ndarray): - raise ValueError('Targets must be of type np.ndarray, but is %s' % - type(targets)) - - filepath = self._get_targets_ensemble_filename() - - # Try to open the file without locking it, this will reduce the - # number of times where we erroneously keep a lock on the ensemble - # targets file although the process already was killed - try: - existing_targets = np.load(filepath, allow_pickle=True) - if existing_targets.shape[0] > targets.shape[0] or ( - existing_targets.shape == targets.shape and np.allclose(existing_targets, targets)): - return filepath - except Exception: - pass - - with lockfile.LockFile(filepath): - if os.path.exists(filepath): - with open(filepath, 'rb') as fh: - existing_targets = np.load(fh, allow_pickle=True) - if existing_targets.shape[0] > targets.shape[0] or ( - existing_targets.shape == targets.shape and np.allclose(existing_targets, targets)): - return filepath - - with tempfile.NamedTemporaryFile('wb', dir=os.path.dirname( - filepath), delete=False) as fh_w: - np.save(fh_w, targets.astype(np.float32)) - tempname = fh_w.name - - os.rename(tempname, filepath) - - return filepath - - def load_targets_ensemble(self) -> np.ndarray: - filepath = self._get_targets_ensemble_filename() - - with lockfile.LockFile(filepath): - with open(filepath, 'rb') as fh: - targets = np.load(fh, allow_pickle=True) - - return targets - - def _get_datamanager_pickle_filename(self) -> str: - return os.path.join(self.internals_directory, 'datamanager.pkl') - - def save_datamanager(self, datamanager: BaseDataset) -> str: - self._make_internals_directory() - filepath = self._get_datamanager_pickle_filename() - - with lockfile.LockFile(filepath): - if not os.path.exists(filepath): - with tempfile.NamedTemporaryFile('wb', dir=os.path.dirname( - filepath), delete=False) as fh: - pickle.dump(datamanager, fh, -1) - tempname = fh.name - os.rename(tempname, filepath) - - return filepath - - def load_datamanager(self) -> BaseDataset: - filepath = self._get_datamanager_pickle_filename() - with lockfile.LockFile(filepath): - with open(filepath, 'rb') as fh: - return pickle.load(fh) - - def replace_datamanager(self, datamanager: BaseDataset) -> None: - """ - This function is called to replace the old datamanager with a datamanager - in case it is required. - - Args: - datamanager (BaseDataset): the new datamanager to replace the old. - """ - warnings.warn("Original dataset will be overwritten with the provided dataset") - datamanager_pickle_file = self._get_datamanager_pickle_filename() - if os.path.exists(datamanager_pickle_file): - os.remove(datamanager_pickle_file) - self.save_datamanager(datamanager=datamanager) - - def get_runs_directory(self) -> str: - return os.path.join(self.internals_directory, 'runs') - - def get_numrun_directory(self, seed: int, num_run: int, budget: float) -> str: - return os.path.join(self.internals_directory, 'runs', '%d_%d_%s' % (seed, num_run, budget)) - - def get_next_num_run(self, peek: bool = False) -> int: - """ - Every pipeline that is fitted by the estimator is stored with an - identifier called num_run. A dummy classifier will always have a num_run - equal to 1, and all other new configurations that are explored will - have a sequentially increasing identifier. - - This method returns the next num_run a configuration should take. - - Parameters - ---------- - peek: bool - By default, the next num_rum will be returned, i.e. self.active_num_run + 1 - Yet, if this bool parameter is equal to True, the value of the current - num_run is provided, i.e, self.active_num_run. - In other words, peek allows to get the current maximum identifier - of a configuration. - - Returns - ------- - num_run: int - An unique identifier for a configuration - """ - - # If there are other num_runs, their name would be runs/__ - other_num_runs = [int(os.path.basename(run_dir).split('_')[1]) - for run_dir in glob.glob(os.path.join(self.internals_directory, 'runs', '*')) - if re.match(r"\d+_\d+_\d+", os.path.basename(run_dir))] - if len(other_num_runs) > 0: - # We track the number of runs from two forefronts: - # The physically available num_runs (which might be deleted or a crash could happen) - # From a internally kept attribute. The later should be sufficient, but we - # want to be robust against multiple backend copies on different workers - self.active_num_run = max([self.active_num_run] + other_num_runs) - - # We are interested in the next run id - if not peek: - self.active_num_run += 1 - return self.active_num_run - - def get_model_filename(self, seed: int, idx: int, budget: float) -> str: - return '%s.%s.%s.model' % (seed, idx, budget) - - def get_cv_model_filename(self, seed: int, idx: int, budget: float) -> str: - return '%s.%s.%s.cv_model' % (seed, idx, budget) - - def list_all_models(self, seed: int) -> List[str]: - runs_directory = self.get_runs_directory() - model_files = glob.glob( - os.path.join(glob.escape(runs_directory), '%d_*' % seed, '%s.*.*.model' % seed) - ) - return model_files - - def load_models_by_identifiers(self, identifiers: List[Tuple[int, int, float]] - ) -> Dict: - models = dict() - - for identifier in identifiers: - seed, idx, budget = identifier - models[identifier] = self.load_model_by_seed_and_id_and_budget( - seed, idx, budget) - - return models - - def load_model_by_seed_and_id_and_budget(self, seed: int, - idx: int, - budget: float - ) -> BasePipeline: - model_directory = self.get_numrun_directory(seed, idx, budget) - - model_file_name = '%s.%s.%s.model' % (seed, idx, budget) - model_file_path = os.path.join(model_directory, model_file_name) - with open(model_file_path, 'rb') as fh: - return pickle.load(fh) - - def load_cv_models_by_identifiers(self, identifiers: List[Tuple[int, int, float]] - ) -> Dict: - models = dict() - - for identifier in identifiers: - seed, idx, budget = identifier - models[identifier] = self.load_cv_model_by_seed_and_id_and_budget( - seed, idx, budget) - - return models - - def load_cv_model_by_seed_and_id_and_budget(self, - seed: int, - idx: int, - budget: float - ) -> BasePipeline: - model_directory = self.get_numrun_directory(seed, idx, budget) - - model_file_name = '%s.%s.%s.cv_model' % (seed, idx, budget) - model_file_path = os.path.join(model_directory, model_file_name) - with open(model_file_path, 'rb') as fh: - return pickle.load(fh) - - def save_numrun_to_dir( - self, seed: int, idx: int, budget: float, model: Optional[BasePipeline], - cv_model: Optional[BasePipeline], ensemble_predictions: Optional[np.ndarray], - valid_predictions: Optional[np.ndarray], test_predictions: Optional[np.ndarray], - ) -> None: - assert self._logger is not None - runs_directory = self.get_runs_directory() - tmpdir = tempfile.mkdtemp(dir=runs_directory) - if model is not None: - file_path = os.path.join(tmpdir, self.get_model_filename(seed, idx, budget)) - with open(file_path, 'wb') as fh: - pickle.dump(model, fh, -1) - - if cv_model is not None: - file_path = os.path.join(tmpdir, self.get_cv_model_filename(seed, idx, budget)) - with open(file_path, 'wb') as fh: - pickle.dump(cv_model, fh, -1) - - for preds, subset in ( - (ensemble_predictions, 'ensemble'), - (valid_predictions, 'valid'), - (test_predictions, 'test') - ): - if preds is not None: - file_path = os.path.join( - tmpdir, - self.get_prediction_filename(subset, seed, idx, budget) - ) - with open(file_path, 'wb') as fh: - pickle.dump(preds.astype(np.float32), fh, -1) - try: - self._logger.debug("Renaming {} to {}".format(tmpdir, - self.get_numrun_directory(seed, idx, budget))) - os.rename(tmpdir, self.get_numrun_directory(seed, idx, budget)) - except OSError: - if os.path.exists(self.get_numrun_directory(seed, idx, budget)): - os.rename(self.get_numrun_directory(seed, idx, budget), - os.path.join(runs_directory, tmpdir + '.old')) - os.rename(tmpdir, self.get_numrun_directory(seed, idx, budget)) - shutil.rmtree(os.path.join(runs_directory, tmpdir + '.old')) - - def get_ensemble_dir(self) -> str: - return os.path.join(self.internals_directory, 'ensembles') - - def load_ensemble(self, seed: int) -> Optional[AbstractEnsemble]: - ensemble_dir = self.get_ensemble_dir() - - if not os.path.exists(ensemble_dir): - if self._logger is not None: - self._logger.warning('Directory %s does not exist' % ensemble_dir) - else: - warnings.warn('Directory %s does not exist' % ensemble_dir) - return None - - if seed >= 0: - indices_files = glob.glob( - os.path.join(glob.escape(ensemble_dir), '%s.*.ensemble' % seed) - ) - indices_files.sort() - else: - indices_files = os.listdir(ensemble_dir) - indices_files = [os.path.join(ensemble_dir, f) for f in indices_files] - indices_files.sort(key=lambda f: time.ctime(os.path.getmtime(f))) - - with open(indices_files[-1], 'rb') as fh: - ensemble_members_run_numbers = pickle.load(fh) - - return ensemble_members_run_numbers - - def save_ensemble(self, ensemble: AbstractEnsemble, idx: int, seed: int) -> None: - try: - os.makedirs(self.get_ensemble_dir()) - except Exception: - pass - - filepath = os.path.join( - self.get_ensemble_dir(), - '%s.%s.ensemble' % (str(seed), str(idx).zfill(10)) - ) - with tempfile.NamedTemporaryFile('wb', dir=os.path.dirname( - filepath), delete=False) as fh: - pickle.dump(ensemble, fh) - tempname = fh.name - os.rename(tempname, filepath) - - def get_prediction_filename(self, subset: str, - automl_seed: Union[str, int], - idx: int, - budget: float - ) -> str: - return 'predictions_%s_%s_%s_%s.npy' % (subset, automl_seed, idx, budget) - - def save_predictions_as_txt(self, - predictions: np.ndarray, - subset: str, - idx: int, precision: int, - prefix: Optional[str] = None) -> None: - if not self.output_directory: - return - # Write prediction scores in prescribed format - filepath = os.path.join( - self.output_directory, - ('%s_' % prefix if prefix else '') + '%s_%s.predict' % (subset, str(idx)), - ) - - format_string = '{:.%dg} ' % precision - with tempfile.NamedTemporaryFile('w', dir=os.path.dirname( - filepath), delete=False) as output_file: - for row in predictions: - if not isinstance(row, np.ndarray) and not isinstance(row, list): - row = [row] - for val in row: - output_file.write(format_string.format(float(val))) - output_file.write('\n') - tempname = output_file.name - os.rename(tempname, filepath) - - def write_txt_file(self, filepath: str, data: str, name: str) -> None: - with lockfile.LockFile(filepath): - with tempfile.NamedTemporaryFile('w', dir=os.path.dirname( - filepath), delete=False) as fh: - fh.write(data) - tempname = fh.name - os.rename(tempname, filepath) - if self._logger is not None: - self._logger.debug('Created %s file %s' % (name, filepath)) diff --git a/examples/40_advanced/40_advanced/example_custom_configuration_space.py b/examples/40_advanced/40_advanced/example_custom_configuration_space.py deleted file mode 100644 index 25eb86be7..000000000 --- a/examples/40_advanced/40_advanced/example_custom_configuration_space.py +++ /dev/null @@ -1,141 +0,0 @@ -""" -====================== -Tabular Classification with Custom Configuration Space -====================== - -The following example shows how adjust the configuration space of -the search. Currently, there are two changes that can be made to the space:- -1. Adjust individual hyperparameters in the pipeline -2. Include or exclude components: - a) include: Dictionary containing components to include. Key is the node - name and Value is an Iterable of the names of the components - to include. Only these components will be present in the - search space. - b) exclude: Dictionary containing components to exclude. Key is the node - name and Value is an Iterable of the names of the components - to exclude. All except these components will be present in - the search space. -""" -import os -import tempfile as tmp -import warnings - -os.environ['JOBLIB_TEMP_FOLDER'] = tmp.gettempdir() -os.environ['OMP_NUM_THREADS'] = '1' -os.environ['OPENBLAS_NUM_THREADS'] = '1' -os.environ['MKL_NUM_THREADS'] = '1' - -warnings.simplefilter(action='ignore', category=UserWarning) -warnings.simplefilter(action='ignore', category=FutureWarning) - -import sklearn.datasets -import sklearn.model_selection - -from autoPyTorch.api.tabular_classification import TabularClassificationTask -from autoPyTorch.utils.hyperparameter_search_space_update import HyperparameterSearchSpaceUpdates - - -def get_search_space_updates(): - """ - Search space updates to the task can be added using HyperparameterSearchSpaceUpdates - Returns: - HyperparameterSearchSpaceUpdates - """ - updates = HyperparameterSearchSpaceUpdates() - updates.append(node_name="data_loader", - hyperparameter="batch_size", - value_range=[16, 512], - default_value=32) - updates.append(node_name="lr_scheduler", - hyperparameter="CosineAnnealingLR:T_max", - value_range=[50, 60], - default_value=55) - updates.append(node_name='network_backbone', - hyperparameter='ResNetBackbone:dropout', - value_range=[0, 0.5], - default_value=0.2) - updates.append(node_name='network_backbone', - hyperparameter='ResNetBackbone:multi_branch_choice', - value_range=['shake-shake'], - default_value='shake-shake') - updates.append(node_name='network_backbone', - hyperparameter='ResNetBackbone:shake_shake_update_func', - value_range=['M3'], - default_value='M3' - ) - return updates - - -if __name__ == '__main__': - - ############################################################################ - # Data Loading - # ============ - X, y = sklearn.datasets.fetch_openml(data_id=40981, return_X_y=True, as_frame=True) - X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split( - X, - y, - random_state=1, - ) - - ############################################################################ - # Build and fit a classifier with include components - # ================================================== - api = TabularClassificationTask( - search_space_updates=get_search_space_updates(), - include_components={'network_backbone': ['ResNetBackbone'], - 'encoder': ['OneHotEncoder']} - ) - - ############################################################################ - # Search for an ensemble of machine learning algorithms - # ===================================================== - api.search( - X_train=X_train.copy(), - y_train=y_train.copy(), - X_test=X_test.copy(), - y_test=y_test.copy(), - optimize_metric='accuracy', - total_walltime_limit=300, - func_eval_time_limit_secs=50 - ) - - ############################################################################ - # Print the final ensemble performance - # ==================================== - print(api.run_history, api.trajectory) - y_pred = api.predict(X_test) - score = api.score(y_pred, y_test) - print(score) - print(api.show_models()) - - ############################################################################ - # Build and fit a classifier with exclude components - # ================================================== - api = TabularClassificationTask( - search_space_updates=get_search_space_updates(), - exclude_components={'network_backbone': ['MLPBackbone'], - 'encoder': ['OneHotEncoder']} - ) - - ############################################################################ - # Search for an ensemble of machine learning algorithms - # ===================================================== - api.search( - X_train=X_train, - y_train=y_train, - X_test=X_test.copy(), - y_test=y_test.copy(), - optimize_metric='accuracy', - total_walltime_limit=300, - func_eval_time_limit_secs=50 - ) - - ############################################################################ - # Print the final ensemble performance - # ==================================== - print(api.run_history, api.trajectory) - y_pred = api.predict(X_test) - score = api.score(y_pred, y_test) - print(score) - print(api.show_models()) diff --git a/examples/40_advanced/example_custom_configuration_space.py b/examples/40_advanced/example_custom_configuration_space.py index 985d9d9ff..25eb86be7 100644 --- a/examples/40_advanced/example_custom_configuration_space.py +++ b/examples/40_advanced/example_custom_configuration_space.py @@ -5,7 +5,6 @@ The following example shows how adjust the configuration space of the search. Currently, there are two changes that can be made to the space:- - 1. Adjust individual hyperparameters in the pipeline 2. Include or exclude components: a) include: Dictionary containing components to include. Key is the node @@ -55,81 +54,88 @@ def get_search_space_updates(): hyperparameter='ResNetBackbone:dropout', value_range=[0, 0.5], default_value=0.2) + updates.append(node_name='network_backbone', + hyperparameter='ResNetBackbone:multi_branch_choice', + value_range=['shake-shake'], + default_value='shake-shake') + updates.append(node_name='network_backbone', + hyperparameter='ResNetBackbone:shake_shake_update_func', + value_range=['M3'], + default_value='M3' + ) return updates -############################################################################ -# Data Loading -# ============ -X, y = sklearn.datasets.fetch_openml(data_id=40981, return_X_y=True, as_frame=True) -X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split( - X, - y, - random_state=1, -) - -############################################################################ -# Build and fit a classifier with include components -# ================================================== -api = TabularClassificationTask( - search_space_updates=get_search_space_updates(), - include_components={'network_backbone': ['MLPBackbone', 'ResNetBackbone'], - 'encoder': ['OneHotEncoder']} -) - -############################################################################ -# Search for an ensemble of machine learning algorithms -# ===================================================== -api.search( - X_train=X_train.copy(), - y_train=y_train.copy(), - X_test=X_test.copy(), - y_test=y_test.copy(), - optimize_metric='accuracy', - total_walltime_limit=150, - func_eval_time_limit_secs=30 -) - -############################################################################ -# Print the final ensemble performance -# ==================================== -y_pred = api.predict(X_test) -score = api.score(y_pred, y_test) -print(score) -print(api.show_models()) - -# Print statistics from search -print(api.sprint_statistics()) - -############################################################################ -# Build and fit a classifier with exclude components -# ================================================== -api = TabularClassificationTask( - search_space_updates=get_search_space_updates(), - exclude_components={'network_backbone': ['MLPBackbone'], - 'encoder': ['OneHotEncoder']} -) - -############################################################################ -# Search for an ensemble of machine learning algorithms -# ===================================================== -api.search( - X_train=X_train, - y_train=y_train, - X_test=X_test.copy(), - y_test=y_test.copy(), - optimize_metric='accuracy', - total_walltime_limit=150, - func_eval_time_limit_secs=30 -) - -############################################################################ -# Print the final ensemble performance -# ==================================== -y_pred = api.predict(X_test) -score = api.score(y_pred, y_test) -print(score) -print(api.show_models()) - -# Print statistics from search -print(api.sprint_statistics()) +if __name__ == '__main__': + + ############################################################################ + # Data Loading + # ============ + X, y = sklearn.datasets.fetch_openml(data_id=40981, return_X_y=True, as_frame=True) + X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split( + X, + y, + random_state=1, + ) + + ############################################################################ + # Build and fit a classifier with include components + # ================================================== + api = TabularClassificationTask( + search_space_updates=get_search_space_updates(), + include_components={'network_backbone': ['ResNetBackbone'], + 'encoder': ['OneHotEncoder']} + ) + + ############################################################################ + # Search for an ensemble of machine learning algorithms + # ===================================================== + api.search( + X_train=X_train.copy(), + y_train=y_train.copy(), + X_test=X_test.copy(), + y_test=y_test.copy(), + optimize_metric='accuracy', + total_walltime_limit=300, + func_eval_time_limit_secs=50 + ) + + ############################################################################ + # Print the final ensemble performance + # ==================================== + print(api.run_history, api.trajectory) + y_pred = api.predict(X_test) + score = api.score(y_pred, y_test) + print(score) + print(api.show_models()) + + ############################################################################ + # Build and fit a classifier with exclude components + # ================================================== + api = TabularClassificationTask( + search_space_updates=get_search_space_updates(), + exclude_components={'network_backbone': ['MLPBackbone'], + 'encoder': ['OneHotEncoder']} + ) + + ############################################################################ + # Search for an ensemble of machine learning algorithms + # ===================================================== + api.search( + X_train=X_train, + y_train=y_train, + X_test=X_test.copy(), + y_test=y_test.copy(), + optimize_metric='accuracy', + total_walltime_limit=300, + func_eval_time_limit_secs=50 + ) + + ############################################################################ + # Print the final ensemble performance + # ==================================== + print(api.run_history, api.trajectory) + y_pred = api.predict(X_test) + score = api.score(y_pred, y_test) + print(score) + print(api.show_models()) diff --git a/examples/40_advanced/40_advanced/example_posthoc_ensemble_fit.py b/examples/40_advanced/example_posthoc_ensemble_fit.py similarity index 100% rename from examples/40_advanced/40_advanced/example_posthoc_ensemble_fit.py rename to examples/40_advanced/example_posthoc_ensemble_fit.py diff --git a/requirements.txt b/requirements.txt index 3f37e131c..2a76f011a 100755 --- a/requirements.txt +++ b/requirements.txt @@ -16,4 +16,4 @@ distributed>=2.2.0 catboost lightgbm flaky -tabulate +tabulate \ No newline at end of file diff --git a/test/test_api/test_api.py b/test/test_api/test_api.py index bf562dcde..c043b2d57 100644 --- a/test/test_api/test_api.py +++ b/test/test_api/test_api.py @@ -156,7 +156,6 @@ def test_tabular_classification(openml_id, resampling_strategy, backend, resampl run_key_model_run_dir, f"{estimator.seed}.{successful_num_run}.{run_key.budget}.cv_model" ) - time.sleep(5) assert os.path.exists(model_file), print_debug_information(estimator) model = estimator._backend.load_cv_model_by_seed_and_id_and_budget( diff --git a/test/test_data/test_feature_validator.py b/test/test_data/test_feature_validator.py index bb4193bdf..499ba37c3 100644 --- a/test/test_data/test_feature_validator.py +++ b/test/test_data/test_feature_validator.py @@ -411,7 +411,6 @@ def test_comparator(): assert ans == feat_type -<<<<<<< HEAD @pytest.fixture def input_data_feature_feat_types(request): if request.param == 'pandas_categoricalonly': @@ -531,8 +530,6 @@ def test_feature_validator_get_columns_to_encode_error_feat_type(input_data_feat with pytest.raises(ValueError, match=r"Expected type of features to be in .*"): validator._validate_feat_types(X) -======= ->>>>>>> [FIX] Passing checks (#298) def test_feature_validator_imbalanced_data(): # Null columns in the train split but not necessarily in the test split @@ -587,14 +584,13 @@ def test_feature_validator_imbalanced_data(): transformed_X_test = validator.transform(X_test) transformed_X_test = pd.DataFrame(transformed_X_test) -<<<<<<< HEAD assert not len(validator.all_nan_columns) + for column in transformed_X_test.columns: + if transformed_X_test[column].isna().all(): + null_columns.append(column) - -<<<<<<< HEAD assert null_columns == [1] ->>>>>>> Fixing issues with imbalanced datasets (#197) -======= + def test_comparator(): numerical = 'numerical' categorical = 'categorical' @@ -616,12 +612,3 @@ def test_comparator(): key=functools.cmp_to_key(validator._comparator) ) assert ans == feat_type ->>>>>>> Bug fixes (#249) -======= - null_columns = [] - for column in transformed_X_test.columns: - if transformed_X_test[column].isna().all(): - null_columns.append(column) - - assert null_columns == [1] ->>>>>>> [FIX] Passing checks (#298) diff --git a/test/test_evaluation/test_fit_evaluator.py b/test/test_evaluation/test_fit_evaluator.py index 4e760a50c..1515ba74f 100644 --- a/test/test_evaluation/test_fit_evaluator.py +++ b/test/test_evaluation/test_fit_evaluator.py @@ -14,12 +14,12 @@ from smac.tae import StatusType +from autoPyTorch.automl_common.common.utils.backend import create from autoPyTorch.datasets.resampling_strategy import NoResamplingStrategyTypes from autoPyTorch.evaluation.fit_evaluator import FitEvaluator from autoPyTorch.evaluation.utils import read_queue from autoPyTorch.pipeline.base_pipeline import BasePipeline from autoPyTorch.pipeline.components.training.metrics.metrics import accuracy -from autoPyTorch.utils import backend this_directory = os.path.dirname(__file__) sys.path.append(this_directory) @@ -93,9 +93,10 @@ def test_no_resampling(self, pipeline_mock): lambda X, batch_size=None: np.tile([0.6, 0.4], (len(X), 1)) pipeline_mock.side_effect = lambda **kwargs: pipeline_mock pipeline_mock.get_additional_run_info.return_value = None + pipeline_mock.get_default_pipeline_options.return_value = {'budget_type': 'epochs', 'epochs': 10} configuration = unittest.mock.Mock(spec=Configuration) - backend_api = backend.create(self.tmp_dir, self.output_dir) + backend_api = create(self.tmp_dir, self.output_dir, 'autoPyTorch') backend_api.load_datamanager = lambda: D queue_ = multiprocessing.Queue() @@ -182,7 +183,7 @@ def test_predict_proba_binary_classification(self, mock): [[0.1, 0.9]] * y.shape[0] ) mock.side_effect = lambda **kwargs: mock - + mock.get_default_pipeline_options.return_value = {'budget_type': 'epochs', 'epochs': 10} configuration = unittest.mock.Mock(spec=Configuration) queue_ = multiprocessing.Queue() diff --git a/test/test_pipeline/components/setup/test_setup.py b/test/test_pipeline/components/setup/test_setup.py index aafc34eb8..72e71a09b 100644 --- a/test/test_pipeline/components/setup/test_setup.py +++ b/test/test_pipeline/components/setup/test_setup.py @@ -445,11 +445,11 @@ def test_add_network_backbone(self): # clear addons base_network_backbone_choice._addons = ThirdPartyComponents(NetworkBackboneComponent) - @pytest.mark.parametrize('resnet_shape', ['funnel', 'long_funnel', - 'diamond', 'hexagon', - 'brick', 'triangle', - 'stairs']) - def test_dropout(self, resnet_shape): + @pytest.mark.parametrize('dropout_shape', ['funnel', 'long_funnel', + 'diamond', 'hexagon', + 'brick', 'triangle', + 'stairs']) + def test_dropout(self, dropout_shape): # ensures that dropout is assigned to the resblock as expected dataset_properties = {"task_type": constants.TASK_TYPES_TO_STRING[1]} max_dropout = 0.5 @@ -463,10 +463,10 @@ def test_dropout(self, resnet_shape): hyperparameter='max_dropout', value_range=[max_dropout], default_value=max_dropout), - resnet_shape=HyperparameterSearchSpace( - hyperparameter='resnet_shape', - value_range=[resnet_shape], - default_value=resnet_shape), + dropout_shape=HyperparameterSearchSpace( + hyperparameter='dropout_shape', + value_range=[dropout_shape], + default_value=dropout_shape), num_groups=HyperparameterSearchSpace( hyperparameter='num_groups', value_range=[num_groups], @@ -481,9 +481,10 @@ def test_dropout(self, resnet_shape): config = config_space.sample_configuration().get_dictionary() resnet_backbone = ShapedResNetBackbone(**config) backbone = resnet_backbone.build_backbone((100, 5)) - dropout_probabilites = [resnet_backbone.config[key] for key in resnet_backbone.config if 'dropout_' in key] + dropout_probabilites = [resnet_backbone.config[key] for key in resnet_backbone.config + if 'dropout_' in key and 'shape' not in key] dropout_shape = get_shaped_neuron_counts( - shape=resnet_shape, + shape=dropout_shape, in_feat=0, out_feat=0, max_neurons=max_dropout, diff --git a/test/test_pipeline/components/training/test_training.py b/test/test_pipeline/components/training/test_training.py index 44a903308..39fa7668e 100644 --- a/test/test_pipeline/components/training/test_training.py +++ b/test/test_pipeline/components/training/test_training.py @@ -21,6 +21,7 @@ ) from autoPyTorch.pipeline.components.training.trainer import ( TrainerChoice, +) from autoPyTorch.pipeline.components.training.trainer.AdversarialTrainer import ( AdversarialTrainer ) @@ -352,6 +353,7 @@ def test_classification_epoch_training(self, n_samples): if counter > epochs: pytest.fail(f"Could not overfit a dummy classification under {epochs} epochs") + def test_every_trainer_is_valid(): """ Makes sure that every trainer is a valid estimator. @@ -517,7 +519,7 @@ def dummy_performance(*args, **kwargs): 'step_interval': StepIntervalUnit.batch } for item in ['backend', 'lr_scheduler', 'network', 'optimizer', 'train_data_loader', 'val_data_loader', - 'device', 'y_train']: + 'device', 'y_train', 'network_snapshots']: fit_dictionary[item] = unittest.mock.MagicMock() fit_dictionary['backend'].temporary_directory = tempfile.mkdtemp() @@ -537,9 +539,9 @@ def dummy_performance(*args, **kwargs): shutil.rmtree(fit_dictionary['backend'].temporary_directory) -class AdversarialTrainerTest(BaseTraining, unittest.TestCase): +class TestAdversarialTrainer(BaseTraining): - def test_epoch_training(self): + def test_epoch_training(self, n_samples): """ Makes sure we are able to train a model and produce good training performance @@ -550,8 +552,10 @@ def test_epoch_training(self): loader, _, epochs, - logger) = self.prepare_trainer(AdversarialTrainer(epsilon=0.07), - constants.TABULAR_CLASSIFICATION) + logger) = self.prepare_trainer(n_samples, + AdversarialTrainer(epsilon=0.07), + constants.TABULAR_CLASSIFICATION, + OVERFIT_EPOCHS) # Train the model counter = 0 diff --git a/test/test_pipeline/test_tabular_classification.py b/test/test_pipeline/test_tabular_classification.py index 29d5dfbaa..e2b14c59f 100644 --- a/test/test_pipeline/test_tabular_classification.py +++ b/test/test_pipeline/test_tabular_classification.py @@ -27,8 +27,10 @@ from autoPyTorch.pipeline.components.training.trainer.utils import Lookahead from autoPyTorch.pipeline.tabular_classification import TabularClassificationPipeline from autoPyTorch.utils.common import FitRequirement -from autoPyTorch.utils.hyperparameter_search_space_update import HyperparameterSearchSpaceUpdates, \ +from autoPyTorch.utils.hyperparameter_search_space_update import ( + HyperparameterSearchSpaceUpdates, parse_hyperparameter_search_space_updates +) @pytest.fixture @@ -573,6 +575,12 @@ def test_train_pipeline_with_runtime(fit_dictionary_tabular_dummy): cs = pipeline.get_hyperparameter_search_space() config = cs.get_default_configuration() + trainer = config.get('trainer:__choice__') + config_dict = config.get_dictionary() + config_dict[f'trainer:{trainer}:use_stochastic_weight_averaging'] = False + config_dict[f'trainer:{trainer}:use_snapshot_ensemble'] = False + del config_dict[f'trainer:{trainer}:se_lastk'] + config = Configuration(cs, values=config_dict) pipeline.set_hyperparameters(config) pipeline.fit(fit_dictionary_tabular_dummy) diff --git a/test/test_pipeline/test_tabular_regression.py b/test/test_pipeline/test_tabular_regression.py index 5889ed1c6..a2c3b695e 100644 --- a/test/test_pipeline/test_tabular_regression.py +++ b/test/test_pipeline/test_tabular_regression.py @@ -20,6 +20,7 @@ from autoPyTorch.pipeline.tabular_regression import TabularRegressionPipeline from autoPyTorch.utils.common import FitRequirement from autoPyTorch.utils.hyperparameter_search_space_update import ( + HyperparameterSearchSpaceUpdate, HyperparameterSearchSpaceUpdates, parse_hyperparameter_search_space_updates ) @@ -317,13 +318,20 @@ def test_pipeline_score(fit_dictionary_tabular_dummy): given the default configuration""" # increase number of epochs to test for performance fit_dictionary_tabular_dummy['epochs'] = 50 - fit_dictionary_tabular_dummy['early_stopping'] = 30 + fit_dictionary_tabular_dummy['early_stopping'] = -1 X = fit_dictionary_tabular_dummy['X_train'].copy() y = fit_dictionary_tabular_dummy['y_train'].copy() pipeline = TabularRegressionPipeline( dataset_properties=fit_dictionary_tabular_dummy['dataset_properties'], + search_space_updates=HyperparameterSearchSpaceUpdates([ + HyperparameterSearchSpaceUpdate("optimizer", + "AdamOptimizer:lr", + value_range=[0.0001, 0.001], + default_value=0.001)] + ), + exclude={'trainer': ['AdversarialTrainer']}, random_state=2 ) @@ -339,5 +347,5 @@ def test_pipeline_score(fit_dictionary_tabular_dummy): r2_score = pipeline.score(X, y) # we should be able to get a decent score on this dummy data - assert r2_score >= 0.8, f"Pipeline:{pipeline} Config:{config} FitDict: {fit_dictionary_tabular_dummy}, " \ + assert r2_score >= 0.5, f"Pipeline:{pipeline} Config:{config} FitDict: {fit_dictionary_tabular_dummy}, " \ f"{pipeline.named_steps['trainer'].run_summary.performance_tracker['train_metrics']}" From 03ddb6410d024c13cb53a2d7aef1ee9dd5b6ab07 Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Tue, 21 Dec 2021 18:01:00 +0100 Subject: [PATCH 39/50] rebase and fix flake --- autoPyTorch/api/base_task.py | 22 +++++- autoPyTorch/api/tabular_classification.py | 3 - autoPyTorch/api/tabular_regression.py | 3 - autoPyTorch/evaluation/fit_evaluator.py | 83 +++++++++++++++++++---- autoPyTorch/evaluation/train_evaluator.py | 2 +- 5 files changed, 93 insertions(+), 20 deletions(-) diff --git a/autoPyTorch/api/base_task.py b/autoPyTorch/api/base_task.py index 37784c6f2..4c516e37b 100644 --- a/autoPyTorch/api/base_task.py +++ b/autoPyTorch/api/base_task.py @@ -306,7 +306,14 @@ def _get_dataset_input_validator( y_train: Union[List, pd.DataFrame, np.ndarray], X_test: Optional[Union[List, pd.DataFrame, np.ndarray]] = None, y_test: Optional[Union[List, pd.DataFrame, np.ndarray]] = None, +<<<<<<< HEAD resampling_strategy: Optional[ResamplingStrategies] = None, +======= + resampling_strategy: Optional[Union[ + CrossValTypes, + HoldoutValTypes, + NoResamplingStrategyTypes]] = None, +>>>>>>> rebase and fix flake resampling_strategy_args: Optional[Dict[str, Any]] = None, dataset_name: Optional[str] = None, dataset_compression: Optional[DatasetCompressionSpec] = None, @@ -353,7 +360,14 @@ def get_dataset( y_train: Union[List, pd.DataFrame, np.ndarray], X_test: Optional[Union[List, pd.DataFrame, np.ndarray]] = None, y_test: Optional[Union[List, pd.DataFrame, np.ndarray]] = None, +<<<<<<< HEAD resampling_strategy: Optional[ResamplingStrategies] = None, +======= + resampling_strategy: Optional[Union[ + CrossValTypes, + HoldoutValTypes, + NoResamplingStrategyTypes]] = None, +>>>>>>> rebase and fix flake resampling_strategy_args: Optional[Dict[str, Any]] = None, dataset_name: Optional[str] = None, dataset_compression: Optional[DatasetCompressionSpec] = None, @@ -1466,7 +1480,14 @@ def fit_pipeline( X_test: Optional[Union[List, pd.DataFrame, np.ndarray]] = None, y_test: Optional[Union[List, pd.DataFrame, np.ndarray]] = None, dataset_name: Optional[str] = None, +<<<<<<< HEAD resampling_strategy: Optional[Union[HoldoutValTypes, CrossValTypes, NoResamplingStrategyTypes]] = None, +======= + resampling_strategy: Optional[Union[ + CrossValTypes, + HoldoutValTypes, + NoResamplingStrategyTypes]] = None, +>>>>>>> rebase and fix flake resampling_strategy_args: Optional[Dict[str, Any]] = None, run_time_limit_secs: int = 60, memory_limit: Optional[int] = None, @@ -1590,7 +1611,6 @@ def fit_pipeline( (BaseDataset): Dataset created from the given tensors """ - self.dataset_name = dataset.dataset_name if dataset is None: if ( diff --git a/autoPyTorch/api/tabular_classification.py b/autoPyTorch/api/tabular_classification.py index fdb1cee09..2ae78342c 100644 --- a/autoPyTorch/api/tabular_classification.py +++ b/autoPyTorch/api/tabular_classification.py @@ -448,9 +448,6 @@ def search( dataset_compression=self._dataset_compression, feat_types=feat_types) - if self.dataset is None: - raise ValueError("`dataset` in {} must be initialized, but got None".format(self.__class__.__name__)) - return self._search( dataset=self.dataset, optimize_metric=optimize_metric, diff --git a/autoPyTorch/api/tabular_regression.py b/autoPyTorch/api/tabular_regression.py index 59b788266..0dd449b60 100644 --- a/autoPyTorch/api/tabular_regression.py +++ b/autoPyTorch/api/tabular_regression.py @@ -448,9 +448,6 @@ def search( dataset_compression=self._dataset_compression, feat_types=feat_types) - if self.dataset is None: - raise ValueError("`dataset` in {} must be initialized, but got None".format(self.__class__.__name__)) - return self._search( dataset=self.dataset, optimize_metric=optimize_metric, diff --git a/autoPyTorch/evaluation/fit_evaluator.py b/autoPyTorch/evaluation/fit_evaluator.py index f171cc18c..52c47b4fa 100644 --- a/autoPyTorch/evaluation/fit_evaluator.py +++ b/autoPyTorch/evaluation/fit_evaluator.py @@ -16,6 +16,7 @@ AbstractEvaluator, fit_and_suppress_warnings ) +from autoPyTorch.evaluation.utils import DisableFileOutputParameters from autoPyTorch.pipeline.components.training.metrics.base import autoPyTorchMetric from autoPyTorch.utils.common import subsampler from autoPyTorch.utils.hyperparameter_search_space_update import HyperparameterSearchSpaceUpdates @@ -33,7 +34,7 @@ def __init__(self, backend: Backend, queue: Queue, num_run: Optional[int] = None, include: Optional[Dict[str, Any]] = None, exclude: Optional[Dict[str, Any]] = None, - disable_file_output: Union[bool, List] = False, + disable_file_output: Optional[List[Union[str, DisableFileOutputParameters]]] = None, init_params: Optional[Dict[str, Any]] = None, logger_port: Optional[int] = None, keep_models: Optional[bool] = None, @@ -241,14 +242,11 @@ def file_output( ) # Abort if we don't want to output anything. - if hasattr(self, 'disable_file_output'): - if self.disable_file_output: - return None, {} - else: - self.disabled_file_outputs = [] + if 'all' in self.disable_file_output: + return None, {} - if hasattr(self, 'pipeline') and self.pipeline is not None: - if 'pipeline' not in self.disabled_file_outputs: + if getattr(self, 'pipeline', None) is not None: + if 'pipeline' not in self.disable_file_output: pipeline = self.pipeline else: pipeline = None @@ -265,11 +263,11 @@ def file_output( ensemble_predictions=None, valid_predictions=( Y_valid_pred if 'y_valid' not in - self.disabled_file_outputs else None + self.disable_file_output else None ), test_predictions=( Y_test_pred if 'y_test' not in - self.disabled_file_outputs else None + self.disable_file_output else None ), ) @@ -287,8 +285,8 @@ def eval_function( num_run: int, include: Optional[Dict[str, Any]], exclude: Optional[Dict[str, Any]], - disable_file_output: Union[bool, List], output_y_hat_optimization: bool = False, + disable_file_output: Optional[List[Union[str, DisableFileOutputParameters]]] = None, pipeline_config: Optional[Dict[str, Any]] = None, budget_type: str = None, init_params: Optional[Dict[str, Any]] = None, @@ -297,6 +295,68 @@ def eval_function( search_space_updates: Optional[HyperparameterSearchSpaceUpdates] = None, instance: str = None, ) -> None: + """ + This closure allows the communication between the ExecuteTaFuncWithQueue and the + pipeline trainer (TrainEvaluator). + + Fundamentally, smac calls the ExecuteTaFuncWithQueue.run() method, which internally + builds a TrainEvaluator. The TrainEvaluator builds a pipeline, stores the output files + to disc via the backend, and puts the performance result of the run in the queue. + + + Attributes: + backend (Backend): + An object to interface with the disk storage. In particular, allows to + access the train and test datasets + queue (Queue): + Each worker available will instantiate an evaluator, and after completion, + it will return the evaluation result via a multiprocessing queue + metric (autoPyTorchMetric): + A scorer object that is able to evaluate how good a pipeline was fit. It + is a wrapper on top of the actual score method (a wrapper on top of scikit + lean accuracy for example) that formats the predictions accordingly. + budget: (float): + The amount of epochs/time a configuration is allowed to run. + budget_type (str): + The budget type, which can be epochs or time + pipeline_config (Optional[Dict[str, Any]]): + Defines the content of the pipeline being evaluated. For example, it + contains pipeline specific settings like logging name, or whether or not + to use tensorboard. + config (Union[int, str, Configuration]): + Determines the pipeline to be constructed. + seed (int): + A integer that allows for reproducibility of results + output_y_hat_optimization (bool): + Whether this worker should output the target predictions, so that they are + stored on disk. Fundamentally, the resampling strategy might shuffle the + Y_train targets, so we store the split in order to re-use them for ensemble + selection. + num_run (Optional[int]): + An identifier of the current configuration being fit. This number is unique per + configuration. + include (Optional[Dict[str, Any]]): + An optional dictionary to include components of the pipeline steps. + exclude (Optional[Dict[str, Any]]): + An optional dictionary to exclude components of the pipeline steps. + disable_file_output (Union[bool, List[str]]): + By default, the model, it's predictions and other metadata is stored on disk + for each finished configuration. This argument allows the user to skip + saving certain file type, for example the model, from being written to disk. + init_params (Optional[Dict[str, Any]]): + Optional argument that is passed to each pipeline step. It is the equivalent of + kwargs for the pipeline steps. + logger_port (Optional[int]): + Logging is performed using a socket-server scheme to be robust against many + parallel entities that want to write to the same file. This integer states the + socket port for the communication channel. If None is provided, a traditional + logger is used. + instance (str): + An instance on which to evaluate the current pipeline. By default we work + with a single instance, being the provided X_train, y_train of a single dataset. + This instance is a compatibility argument for SMAC, that is capable of working + with multiple datasets at the same time. + """ evaluator = FitEvaluator( backend=backend, queue=queue, @@ -304,7 +364,6 @@ def eval_function( configuration=config, seed=seed, num_run=num_run, - output_y_hat_optimization=output_y_hat_optimization, include=include, exclude=exclude, disable_file_output=disable_file_output, diff --git a/autoPyTorch/evaluation/train_evaluator.py b/autoPyTorch/evaluation/train_evaluator.py index 03ff69c32..01a1e3f18 100644 --- a/autoPyTorch/evaluation/train_evaluator.py +++ b/autoPyTorch/evaluation/train_evaluator.py @@ -424,10 +424,10 @@ def eval_train_function( budget: float, config: Optional[Configuration], seed: int, - output_y_hat_optimization: bool, num_run: int, include: Optional[Dict[str, Any]], exclude: Optional[Dict[str, Any]], + output_y_hat_optimization: bool, disable_file_output: Optional[List[Union[str, DisableFileOutputParameters]]] = None, pipeline_config: Optional[Dict[str, Any]] = None, budget_type: str = None, From 59b5830db0156c7f91ef3c7edd879641a1f9d1af Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Fri, 28 Jan 2022 13:44:15 +0100 Subject: [PATCH 40/50] fix merge conflicts after rebase --- autoPyTorch/api/base_task.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/autoPyTorch/api/base_task.py b/autoPyTorch/api/base_task.py index 4c516e37b..2fcf66bae 100644 --- a/autoPyTorch/api/base_task.py +++ b/autoPyTorch/api/base_task.py @@ -306,14 +306,7 @@ def _get_dataset_input_validator( y_train: Union[List, pd.DataFrame, np.ndarray], X_test: Optional[Union[List, pd.DataFrame, np.ndarray]] = None, y_test: Optional[Union[List, pd.DataFrame, np.ndarray]] = None, -<<<<<<< HEAD resampling_strategy: Optional[ResamplingStrategies] = None, -======= - resampling_strategy: Optional[Union[ - CrossValTypes, - HoldoutValTypes, - NoResamplingStrategyTypes]] = None, ->>>>>>> rebase and fix flake resampling_strategy_args: Optional[Dict[str, Any]] = None, dataset_name: Optional[str] = None, dataset_compression: Optional[DatasetCompressionSpec] = None, @@ -360,14 +353,7 @@ def get_dataset( y_train: Union[List, pd.DataFrame, np.ndarray], X_test: Optional[Union[List, pd.DataFrame, np.ndarray]] = None, y_test: Optional[Union[List, pd.DataFrame, np.ndarray]] = None, -<<<<<<< HEAD resampling_strategy: Optional[ResamplingStrategies] = None, -======= - resampling_strategy: Optional[Union[ - CrossValTypes, - HoldoutValTypes, - NoResamplingStrategyTypes]] = None, ->>>>>>> rebase and fix flake resampling_strategy_args: Optional[Dict[str, Any]] = None, dataset_name: Optional[str] = None, dataset_compression: Optional[DatasetCompressionSpec] = None, @@ -1480,14 +1466,7 @@ def fit_pipeline( X_test: Optional[Union[List, pd.DataFrame, np.ndarray]] = None, y_test: Optional[Union[List, pd.DataFrame, np.ndarray]] = None, dataset_name: Optional[str] = None, -<<<<<<< HEAD resampling_strategy: Optional[Union[HoldoutValTypes, CrossValTypes, NoResamplingStrategyTypes]] = None, -======= - resampling_strategy: Optional[Union[ - CrossValTypes, - HoldoutValTypes, - NoResamplingStrategyTypes]] = None, ->>>>>>> rebase and fix flake resampling_strategy_args: Optional[Dict[str, Any]] = None, run_time_limit_secs: int = 60, memory_limit: Optional[int] = None, From c3b8844c57b14fd0caac2322c4518381bcb80175 Mon Sep 17 00:00:00 2001 From: Ravin Kohli <13005107+ravinkohli@users.noreply.github.com> Date: Thu, 10 Feb 2022 00:14:37 +0100 Subject: [PATCH 41/50] [FIX] Enable preprocessing in reg_cocktails (#369) * enable preprocessing and remove is_small_preprocess * address comments from shuhei and fix precommit checks * fix tests * fix precommit checks * add suggestions from shuhei for astype use * address speed issue when using object_dtype_mapping * make code more readable * improve documentation for base network embedding --- autoPyTorch/api/tabular_classification.py | 1 - autoPyTorch/api/tabular_regression.py | 1 - autoPyTorch/data/tabular_feature_validator.py | 137 ++++--- autoPyTorch/datasets/base_dataset.py | 5 - autoPyTorch/evaluation/fit_evaluator.py | 378 ------------------ autoPyTorch/evaluation/train_evaluator.py | 9 +- .../TabularColumnTransformer.py | 4 +- .../encoding/NoEncoder.py | 2 +- .../encoding/base_encoder.py | 2 +- .../imputation/base_imputer.py | 2 +- .../tabular_preprocessing/scaling/NoScaler.py | 2 +- .../scaling/base_scaler.py | 2 +- .../early_preprocessor/EarlyPreprocessing.py | 16 +- .../setup/lr_scheduler/base_scheduler.py | 2 +- .../components/setup/network/base_network.py | 3 +- .../network_backbone/base_network_backbone.py | 8 +- .../base_network_embedding.py | 51 ++- .../training/data_loader/base_data_loader.py | 15 +- .../data_loader/feature_data_loader.py | 4 +- .../training/data_loader/image_data_loader.py | 4 +- .../training/trainer/AdversarialTrainer.py | 2 +- .../components/training/trainer/__init__.py | 2 +- .../training/trainer/base_trainer.py | 1 - test/test_data/test_feature_validator.py | 127 +++++- test/test_data/test_validation.py | 2 +- test/test_datasets/test_tabular_dataset.py | 1 - test/test_evaluation/test_fit_evaluator.py | 206 ---------- .../components/preprocessing/test_encoders.py | 2 - .../components/preprocessing/test_imputers.py | 2 - .../components/preprocessing/test_scalers.py | 8 - .../test_tabular_column_transformer.py | 9 +- .../training/test_feature_data_loader.py | 20 +- .../components/training/test_training.py | 6 - 33 files changed, 266 insertions(+), 770 deletions(-) delete mode 100644 autoPyTorch/evaluation/fit_evaluator.py delete mode 100644 test/test_evaluation/test_fit_evaluator.py diff --git a/autoPyTorch/api/tabular_classification.py b/autoPyTorch/api/tabular_classification.py index 2ae78342c..3ccfa88ea 100644 --- a/autoPyTorch/api/tabular_classification.py +++ b/autoPyTorch/api/tabular_classification.py @@ -18,7 +18,6 @@ from autoPyTorch.datasets.base_dataset import BaseDatasetPropertiesType from autoPyTorch.datasets.resampling_strategy import ( HoldoutValTypes, - CrossValTypes, ResamplingStrategies, ) from autoPyTorch.datasets.tabular_dataset import TabularDataset diff --git a/autoPyTorch/api/tabular_regression.py b/autoPyTorch/api/tabular_regression.py index 0dd449b60..d2b087ddb 100644 --- a/autoPyTorch/api/tabular_regression.py +++ b/autoPyTorch/api/tabular_regression.py @@ -18,7 +18,6 @@ from autoPyTorch.datasets.base_dataset import BaseDatasetPropertiesType from autoPyTorch.datasets.resampling_strategy import ( HoldoutValTypes, - CrossValTypes, ResamplingStrategies, ) from autoPyTorch.datasets.tabular_dataset import TabularDataset diff --git a/autoPyTorch/data/tabular_feature_validator.py b/autoPyTorch/data/tabular_feature_validator.py index f1170f1a2..e38a7eb22 100644 --- a/autoPyTorch/data/tabular_feature_validator.py +++ b/autoPyTorch/data/tabular_feature_validator.py @@ -16,7 +16,7 @@ from sklearn.exceptions import NotFittedError from sklearn.impute import SimpleImputer from sklearn.pipeline import make_pipeline -from sklearn.preprocessing import OneHotEncoder, StandardScaler +from sklearn.preprocessing import OrdinalEncoder from autoPyTorch.data.base_feature_validator import BaseFeatureValidator, SupportedFeatTypes from autoPyTorch.utils.common import ispandas @@ -25,7 +25,6 @@ def _create_column_transformer( preprocessors: Dict[str, List[BaseEstimator]], - numerical_columns: List[str], categorical_columns: List[str], ) -> ColumnTransformer: """ @@ -36,8 +35,6 @@ def _create_column_transformer( Args: preprocessors (Dict[str, List[BaseEstimator]]): Dictionary containing list of numerical and categorical preprocessors. - numerical_columns (List[str]): - List of names of numerical columns categorical_columns (List[str]): List of names of categorical columns @@ -45,17 +42,11 @@ def _create_column_transformer( ColumnTransformer """ - numerical_pipeline = 'drop' - categorical_pipeline = 'drop' - if len(numerical_columns) > 0: - numerical_pipeline = make_pipeline(*preprocessors['numerical']) - if len(categorical_columns) > 0: - categorical_pipeline = make_pipeline(*preprocessors['categorical']) + categorical_pipeline = make_pipeline(*preprocessors['categorical']) return ColumnTransformer([ - ('categorical_pipeline', categorical_pipeline, categorical_columns), - ('numerical_pipeline', numerical_pipeline, numerical_columns)], - remainder='drop' + ('categorical_pipeline', categorical_pipeline, categorical_columns)], + remainder='passthrough' ) @@ -63,22 +54,17 @@ def get_tabular_preprocessors() -> Dict[str, List[BaseEstimator]]: """ This function creates a Dictionary containing a list of numerical and categorical preprocessors - Returns: Dict[str, List[BaseEstimator]] """ preprocessors: Dict[str, List[BaseEstimator]] = dict() # Categorical Preprocessors - onehot_encoder = OneHotEncoder(categories='auto', sparse=False, handle_unknown='ignore') + ordinal_encoder = OrdinalEncoder(handle_unknown='use_encoded_value', + unknown_value=-1) categorical_imputer = SimpleImputer(strategy='constant', copy=False) - # Numerical Preprocessors - numerical_imputer = SimpleImputer(strategy='median', copy=False) - standard_scaler = StandardScaler(with_mean=True, with_std=True, copy=False) - - preprocessors['categorical'] = [categorical_imputer, onehot_encoder] - preprocessors['numerical'] = [numerical_imputer, standard_scaler] + preprocessors['categorical'] = [categorical_imputer, ordinal_encoder] return preprocessors @@ -176,7 +162,16 @@ def _fit( if ispandas(X) and not issparse(X): X = cast(pd.DataFrame, X) - self.all_nan_columns = set([column for column in X.columns if X[column].isna().all()]) + all_nan_columns = X.columns[X.isna().all()] + for col in all_nan_columns: + X[col] = pd.to_numeric(X[col]) + + # Handle objects if possible + exist_object_columns = has_object_columns(X.dtypes.values) + if exist_object_columns: + X = self.infer_objects(X) + self.dtypes = [dt.name for dt in X.dtypes] # Also note this change in self.dtypes + self.all_nan_columns = set(all_nan_columns) self.transformed_columns, self.feat_types = self.get_columns_to_encode(X) @@ -188,18 +183,33 @@ def _fit( categorical_columns=self.transformed_columns, ) - # Mypy redefinition - assert self.column_transformer is not None - self.column_transformer.fit(X) + if len(self.enc_columns) > 0: - # The column transformer reorders the feature types - # therefore, we need to change the order of columns as well - # This means categorical columns are shifted to the left + preprocessors = get_tabular_preprocessors() + self.column_transformer = _create_column_transformer( + preprocessors=preprocessors, + categorical_columns=self.enc_columns, + ) - self.feat_types = sorted( - self.feat_types, - key=functools.cmp_to_key(self._comparator) - ) + # Mypy redefinition + assert self.column_transformer is not None + self.column_transformer.fit(X) + + # The column transformer moves categorical columns before all numerical columns + # therefore, we need to sort categorical columns so that it complies this change + + self.feat_types = sorted( + self.feat_types, + key=functools.cmp_to_key(self._comparator) + ) + + encoded_categories = self.column_transformer.\ + named_transformers_['categorical_pipeline'].\ + named_steps['ordinalencoder'].categories_ + self.categories = [ + list(range(len(cat))) + for cat in encoded_categories + ] # differently to categorical_columns and numerical_columns, # this saves the index of the column. @@ -279,6 +289,23 @@ def transform( if ispandas(X) and not issparse(X): X = cast(pd.DataFrame, X) + if self.all_nan_columns is None: + raise ValueError('_fit must be called before calling transform') + + for col in list(self.all_nan_columns): + X[col] = np.nan + X[col] = pd.to_numeric(X[col]) + + if len(self.categorical_columns) > 0: + # when some categorical columns are not all nan in the training set + # but they are all nan in the testing or validation set + # we change those columns to `object` dtype + # to ensure that these columns are changed to appropriate dtype + # in self.infer_objects + all_nan_cat_cols = set(X[self.enc_columns].columns[X[self.enc_columns].isna().all()]) + dtype_dict = {col: 'object' for col in self.enc_columns if col in all_nan_cat_cols} + X = X.astype(dtype_dict) + # Check the data here so we catch problems on new test data self._check_data(X) @@ -287,11 +314,6 @@ def transform( # We need to convert the column in test data to # object otherwise the test column is interpreted as float if self.column_transformer is not None: - if len(self.categorical_columns) > 0: - categorical_columns = self.column_transformer.transformers_[0][-1] - for column in categorical_columns: - if X[column].isna().all(): - X[column] = X[column].astype('object') X = self.column_transformer.transform(X) # Sparse related transformations @@ -380,7 +402,6 @@ def _check_data( self.column_order = column_order dtypes = [dtype.name for dtype in X.dtypes] - diff_cols = X.columns[[s_dtype != dtype for s_dtype, dtype in zip(self.dtypes, dtypes)]] if len(self.dtypes) == 0: self.dtypes = dtypes @@ -448,7 +469,7 @@ def _validate_feat_types(self, X: pd.DataFrame) -> None: def _get_columns_to_encode( self, X: pd.DataFrame, - ) -> Tuple[List[str], List[str], List[str]]: + ) -> Tuple[List[str], List[str]]: """ Return the columns to be transformed as well as the type of feature for each column from a pandas dataframe. @@ -478,8 +499,8 @@ def _get_columns_to_encode( # Also, register the feature types for the estimator feat_types = [] - # Make sure each column is a valid type - for column in X.columns: + # Make sure each column is a valid type + for i, column in enumerate(X.columns): if self.all_nan_columns is not None and column in self.all_nan_columns: continue column_dtype = self.dtypes[i] @@ -592,22 +613,26 @@ def infer_objects(self, X: pd.DataFrame) -> pd.DataFrame: pd.DataFrame """ if hasattr(self, 'object_dtype_mapping'): - # Mypy does not process the has attr. This dict is defined below - for key, dtype in self.object_dtype_mapping.items(): # type: ignore[has-type] - # honor the training data types - try: - X[key] = X[key].astype(dtype.name) - except Exception as e: - # Try inference if possible - self.logger.warning(f'Casting the column {key} to {dtype} caused the exception {e}') - pass + # honor the training data types + try: + # Mypy does not process the has attr. + X = X.astype(self.object_dtype_mapping) # type: ignore[has-type] + except Exception as e: + # Try inference if possible + self.logger.warning(f'Casting the columns to training dtypes ' # type: ignore[has-type] + f'{self.object_dtype_mapping} caused the exception {e}') + pass else: - # Calling for the first time to infer the categories - X = X.infer_objects() - for column, data_type in zip(X.columns, X.dtypes): - if not is_numeric_dtype(data_type): - X[column] = X[column].astype('category') - + if len(self.dtypes) != 0: + # when train data has no object dtype, but test does + # we prioritise the datatype given in training data + dtype_dict = {col: dtype for col, dtype in zip(X.columns, self.dtypes)} + X = X.astype(dtype_dict) + else: + # Calling for the first time to infer the categories + X = X.infer_objects() + dtype_dict = {col: 'category' for col, dtype in zip(X.columns, X.dtypes) if not is_numeric_dtype(dtype)} + X = X.astype(dtype_dict) # only numerical attributes and categories self.object_dtype_mapping = {column: data_type for column, data_type in zip(X.columns, X.dtypes)} diff --git a/autoPyTorch/datasets/base_dataset.py b/autoPyTorch/datasets/base_dataset.py index a63e2b108..43754d9d7 100644 --- a/autoPyTorch/datasets/base_dataset.py +++ b/autoPyTorch/datasets/base_dataset.py @@ -155,7 +155,6 @@ def __init__( self.holdout_validators: Dict[str, HoldOutFunc] = {} self.no_resampling_validators: Dict[str, NoResamplingFunc] = {} self.random_state = np.random.RandomState(seed=seed) - self.no_resampling_validators: Dict[str, NoResamplingFunc] = {} self.shuffle = shuffle self.resampling_strategy = resampling_strategy self.resampling_strategy_args = resampling_strategy_args @@ -165,10 +164,6 @@ def __init__( if len(self.train_tensors) == 2 and self.train_tensors[1] is not None: self.output_shape, self.output_type = _get_output_properties(self.train_tensors) - # TODO: Look for a criteria to define small enough to preprocess - # False for the regularization cocktails initially - self.is_small_preprocess = False - # Make sure cross validation splits are created once self.cross_validators = CrossValFuncs.get_cross_validators(*CrossValTypes) self.holdout_validators = HoldOutFuncs.get_holdout_validators(*HoldoutValTypes) diff --git a/autoPyTorch/evaluation/fit_evaluator.py b/autoPyTorch/evaluation/fit_evaluator.py deleted file mode 100644 index 52c47b4fa..000000000 --- a/autoPyTorch/evaluation/fit_evaluator.py +++ /dev/null @@ -1,378 +0,0 @@ -import time -from multiprocessing.queues import Queue -from typing import Any, Dict, List, Optional, Tuple, Union - -from ConfigSpace.configuration_space import Configuration - -import numpy as np - -from sklearn.base import BaseEstimator - -from smac.tae import StatusType - -from autoPyTorch.automl_common.common.utils.backend import Backend -from autoPyTorch.datasets.resampling_strategy import NoResamplingStrategyTypes -from autoPyTorch.evaluation.abstract_evaluator import ( - AbstractEvaluator, - fit_and_suppress_warnings -) -from autoPyTorch.evaluation.utils import DisableFileOutputParameters -from autoPyTorch.pipeline.components.training.metrics.base import autoPyTorchMetric -from autoPyTorch.utils.common import subsampler -from autoPyTorch.utils.hyperparameter_search_space_update import HyperparameterSearchSpaceUpdates - - -class FitEvaluator(AbstractEvaluator): - def __init__(self, backend: Backend, queue: Queue, - metric: autoPyTorchMetric, - budget: float, - budget_type: str = None, - pipeline_config: Optional[Dict[str, Any]] = None, - configuration: Optional[Configuration] = None, - seed: int = 1, - output_y_hat_optimization: bool = False, - num_run: Optional[int] = None, - include: Optional[Dict[str, Any]] = None, - exclude: Optional[Dict[str, Any]] = None, - disable_file_output: Optional[List[Union[str, DisableFileOutputParameters]]] = None, - init_params: Optional[Dict[str, Any]] = None, - logger_port: Optional[int] = None, - keep_models: Optional[bool] = None, - all_supported_metrics: bool = True, - search_space_updates: Optional[HyperparameterSearchSpaceUpdates] = None) -> None: - super().__init__( - backend=backend, - queue=queue, - configuration=configuration, - metric=metric, - seed=seed, - output_y_hat_optimization=output_y_hat_optimization, - num_run=num_run, - include=include, - exclude=exclude, - disable_file_output=disable_file_output, - init_params=init_params, - budget=budget, - budget_type=budget_type, - logger_port=logger_port, - all_supported_metrics=all_supported_metrics, - pipeline_config=pipeline_config, - search_space_updates=search_space_updates - ) - if not isinstance(self.datamanager.resampling_strategy, NoResamplingStrategyTypes): - raise ValueError( - "FitEvaluator needs to be fitted on the whole dataset and resampling_strategy " - "must be `NoResamplingStrategyTypes`, but got {}".format( - self.datamanager.resampling_strategy - )) - - self.splits = self.datamanager.splits - self.Y_target: Optional[np.ndarray] = None - self.Y_train_targets: np.ndarray = np.ones(self.y_train.shape) * np.NaN - self.pipeline: Optional[BaseEstimator] = None - - self.logger.debug("Search space updates :{}".format(self.search_space_updates)) - self.keep_models = keep_models - - def fit_predict_and_loss(self) -> None: - """Fit, predict and compute the loss for no resampling strategy""" - assert self.splits is not None, "Can't fit pipeline in {} is datamanager.splits is None" \ - .format(self.__class__.__name__) - additional_run_info: Optional[Dict] = None - split_id = 0 - self.logger.info("Starting fit {}".format(split_id)) - - pipeline = self._get_pipeline() - - train_split, test_split = self.splits[split_id] - assert test_split is None - self.Y_actual_train = self.y_train[train_split] - y_train_pred, y_valid_pred, y_test_pred = self._fit_and_predict(pipeline, split_id, - train_indices=train_split, - test_indices=test_split, - add_pipeline_to_self=True) - train_loss = self._loss(self.y_train[train_split], y_train_pred) - if y_valid_pred is not None: - loss = self._loss(self.y_valid, y_valid_pred) - elif y_test_pred is not None: - loss = self._loss(self.y_test, y_test_pred) - else: - loss = train_loss - - additional_run_info = pipeline.get_additional_run_info() if hasattr( - pipeline, 'get_additional_run_info') else {} - - status = StatusType.SUCCESS - - self.logger.debug("In train evaluator fit_predict_and_loss, num_run: {} loss:{}".format( - self.num_run, - loss - )) - self.finish_up( - loss=loss, - train_loss=train_loss, - valid_pred=y_valid_pred, - test_pred=y_test_pred, - additional_run_info=additional_run_info, - file_output=True, - status=status, - opt_pred=None - ) - - def _fit_and_predict(self, pipeline: BaseEstimator, fold: int, train_indices: Union[np.ndarray, List], - test_indices: None, - add_pipeline_to_self: bool - ) -> Tuple[np.ndarray, Optional[np.ndarray], Optional[np.ndarray]]: - - X = {'train_indices': train_indices, - 'val_indices': test_indices, - 'split_id': fold, - 'num_run': self.num_run, - **self.fit_dictionary} # fit dictionary - y = None - fit_and_suppress_warnings(self.logger, pipeline, X, y) - self.logger.info("Model fitted, now predicting") - ( - Y_train_pred, - Y_valid_pred, - Y_test_pred - ) = self._predict( - pipeline, - train_indices=train_indices, - ) - - if add_pipeline_to_self: - self.pipeline = pipeline - - return Y_train_pred, Y_valid_pred, Y_test_pred - - def _predict(self, pipeline: BaseEstimator, - train_indices: Union[np.ndarray, List] - ) -> Tuple[np.ndarray, Optional[np.ndarray], Optional[np.ndarray]]: - - train_pred = self.predict_function(subsampler(self.X_train, train_indices), pipeline, - self.y_train[train_indices]) - - if self.X_valid is not None: - valid_pred = self.predict_function(self.X_valid, pipeline, - self.y_valid) - else: - valid_pred = None - - if self.X_test is not None: - test_pred = self.predict_function(self.X_test, pipeline, - self.y_train[train_indices]) - else: - test_pred = None - - return train_pred, valid_pred, test_pred - - def finish_up(self, loss: Dict[str, float], train_loss: Dict[str, float], - valid_pred: Optional[np.ndarray], - test_pred: Optional[np.ndarray], additional_run_info: Optional[Dict], - file_output: bool, status: StatusType, - opt_pred: Optional[np.ndarray] - ) -> Optional[Tuple[float, float, int, Dict]]: - """This function does everything necessary after the fitting is done: - - * predicting - * saving the necessary files - We use it as the signal handler so we can recycle the code for the - normal usecase and when the runsolver kills us here :)""" - - self.duration = time.time() - self.starttime - - if file_output: - loss_, additional_run_info_ = self.file_output( - None, valid_pred, test_pred, - ) - else: - loss_ = None - additional_run_info_ = {} - - validation_loss, test_loss = self.calculate_auxiliary_losses( - valid_pred, test_pred - ) - - if loss_ is not None: - return self.duration, loss_, self.seed, additional_run_info_ - - cost = loss[self.metric.name] - - additional_run_info = ( - {} if additional_run_info is None else additional_run_info - ) - for metric_name, value in loss.items(): - additional_run_info[metric_name] = value - additional_run_info['duration'] = self.duration - additional_run_info['num_run'] = self.num_run - if train_loss is not None: - additional_run_info['train_loss'] = train_loss - if validation_loss is not None: - additional_run_info['validation_loss'] = validation_loss - if test_loss is not None: - additional_run_info['test_loss'] = test_loss - - rval_dict = {'loss': cost, - 'additional_run_info': additional_run_info, - 'status': status} - - self.queue.put(rval_dict) - return None - - def file_output( - self, - Y_optimization_pred: np.ndarray, - Y_valid_pred: np.ndarray, - Y_test_pred: np.ndarray, - ) -> Tuple[Optional[float], Dict]: - - # Abort if predictions contain NaNs - for y, s in [ - [Y_valid_pred, 'validation'], - [Y_test_pred, 'test'] - ]: - if y is not None and not np.all(np.isfinite(y)): - return ( - 1.0, - { - 'error': - 'Model predictions for %s set contains NaNs.' % s - }, - ) - - # Abort if we don't want to output anything. - if 'all' in self.disable_file_output: - return None, {} - - if getattr(self, 'pipeline', None) is not None: - if 'pipeline' not in self.disable_file_output: - pipeline = self.pipeline - else: - pipeline = None - else: - pipeline = None - - self.logger.debug("Saving model {}_{}_{} to disk".format(self.seed, self.num_run, self.budget)) - self.backend.save_numrun_to_dir( - seed=int(self.seed), - idx=int(self.num_run), - budget=float(self.budget), - model=pipeline, - cv_model=None, - ensemble_predictions=None, - valid_predictions=( - Y_valid_pred if 'y_valid' not in - self.disable_file_output else None - ), - test_predictions=( - Y_test_pred if 'y_test' not in - self.disable_file_output else None - ), - ) - - return None, {} - - -# create closure for evaluating an algorithm -def eval_function( - backend: Backend, - queue: Queue, - metric: autoPyTorchMetric, - budget: float, - config: Optional[Configuration], - seed: int, - num_run: int, - include: Optional[Dict[str, Any]], - exclude: Optional[Dict[str, Any]], - output_y_hat_optimization: bool = False, - disable_file_output: Optional[List[Union[str, DisableFileOutputParameters]]] = None, - pipeline_config: Optional[Dict[str, Any]] = None, - budget_type: str = None, - init_params: Optional[Dict[str, Any]] = None, - logger_port: Optional[int] = None, - all_supported_metrics: bool = True, - search_space_updates: Optional[HyperparameterSearchSpaceUpdates] = None, - instance: str = None, -) -> None: - """ - This closure allows the communication between the ExecuteTaFuncWithQueue and the - pipeline trainer (TrainEvaluator). - - Fundamentally, smac calls the ExecuteTaFuncWithQueue.run() method, which internally - builds a TrainEvaluator. The TrainEvaluator builds a pipeline, stores the output files - to disc via the backend, and puts the performance result of the run in the queue. - - - Attributes: - backend (Backend): - An object to interface with the disk storage. In particular, allows to - access the train and test datasets - queue (Queue): - Each worker available will instantiate an evaluator, and after completion, - it will return the evaluation result via a multiprocessing queue - metric (autoPyTorchMetric): - A scorer object that is able to evaluate how good a pipeline was fit. It - is a wrapper on top of the actual score method (a wrapper on top of scikit - lean accuracy for example) that formats the predictions accordingly. - budget: (float): - The amount of epochs/time a configuration is allowed to run. - budget_type (str): - The budget type, which can be epochs or time - pipeline_config (Optional[Dict[str, Any]]): - Defines the content of the pipeline being evaluated. For example, it - contains pipeline specific settings like logging name, or whether or not - to use tensorboard. - config (Union[int, str, Configuration]): - Determines the pipeline to be constructed. - seed (int): - A integer that allows for reproducibility of results - output_y_hat_optimization (bool): - Whether this worker should output the target predictions, so that they are - stored on disk. Fundamentally, the resampling strategy might shuffle the - Y_train targets, so we store the split in order to re-use them for ensemble - selection. - num_run (Optional[int]): - An identifier of the current configuration being fit. This number is unique per - configuration. - include (Optional[Dict[str, Any]]): - An optional dictionary to include components of the pipeline steps. - exclude (Optional[Dict[str, Any]]): - An optional dictionary to exclude components of the pipeline steps. - disable_file_output (Union[bool, List[str]]): - By default, the model, it's predictions and other metadata is stored on disk - for each finished configuration. This argument allows the user to skip - saving certain file type, for example the model, from being written to disk. - init_params (Optional[Dict[str, Any]]): - Optional argument that is passed to each pipeline step. It is the equivalent of - kwargs for the pipeline steps. - logger_port (Optional[int]): - Logging is performed using a socket-server scheme to be robust against many - parallel entities that want to write to the same file. This integer states the - socket port for the communication channel. If None is provided, a traditional - logger is used. - instance (str): - An instance on which to evaluate the current pipeline. By default we work - with a single instance, being the provided X_train, y_train of a single dataset. - This instance is a compatibility argument for SMAC, that is capable of working - with multiple datasets at the same time. - """ - evaluator = FitEvaluator( - backend=backend, - queue=queue, - metric=metric, - configuration=config, - seed=seed, - num_run=num_run, - include=include, - exclude=exclude, - disable_file_output=disable_file_output, - init_params=init_params, - budget=budget, - budget_type=budget_type, - logger_port=logger_port, - all_supported_metrics=all_supported_metrics, - pipeline_config=pipeline_config, - search_space_updates=search_space_updates - ) - evaluator.fit_predict_and_loss() diff --git a/autoPyTorch/evaluation/train_evaluator.py b/autoPyTorch/evaluation/train_evaluator.py index 01a1e3f18..f57d5b15a 100644 --- a/autoPyTorch/evaluation/train_evaluator.py +++ b/autoPyTorch/evaluation/train_evaluator.py @@ -153,13 +153,6 @@ def __init__(self, backend: Backend, queue: Queue, search_space_updates=search_space_updates ) - if not isinstance(self.datamanager.resampling_strategy, (CrossValTypes, HoldoutValTypes)): - raise ValueError( - 'TrainEvaluator expect to have (CrossValTypes, HoldoutValTypes) as ' - 'resampling_strategy, but got {}'.format(self.datamanager.resampling_strategy) - ) - - if not isinstance(self.resampling_strategy, (CrossValTypes, HoldoutValTypes)): raise ValueError( f'resampling_strategy for TrainEvaluator must be in ' @@ -424,10 +417,10 @@ def eval_train_function( budget: float, config: Optional[Configuration], seed: int, + output_y_hat_optimization: bool, num_run: int, include: Optional[Dict[str, Any]], exclude: Optional[Dict[str, Any]], - output_y_hat_optimization: bool, disable_file_output: Optional[List[Union[str, DisableFileOutputParameters]]] = None, pipeline_config: Optional[Dict[str, Any]] = None, budget_type: str = None, diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/TabularColumnTransformer.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/TabularColumnTransformer.py index b8805c809..6b38b4650 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/TabularColumnTransformer.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/TabularColumnTransformer.py @@ -4,14 +4,14 @@ from sklearn.base import BaseEstimator from sklearn.compose import ColumnTransformer -# from sklearn.pipeline import make_pipeline +from sklearn.pipeline import make_pipeline import torch from autoPyTorch.pipeline.components.preprocessing.tabular_preprocessing.base_tabular_preprocessing import ( autoPyTorchTabularPreprocessingComponent ) -# from autoPyTorch.pipeline.components.preprocessing.tabular_preprocessing.utils import get_tabular_preprocessers +from autoPyTorch.pipeline.components.preprocessing.tabular_preprocessing.utils import get_tabular_preprocessers from autoPyTorch.utils.common import FitRequirement, subsampler diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/encoding/NoEncoder.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/encoding/NoEncoder.py index d62ee26d2..929e99048 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/encoding/NoEncoder.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/encoding/NoEncoder.py @@ -40,7 +40,7 @@ def transform(self, X: Dict[str, Any]) -> Dict[str, Any]: Returns: (Dict[str, Any]): the updated 'X' dictionary """ - # X.update({'encoder': self.preprocessor}) + X.update({'encoder': self.preprocessor}) return X @staticmethod diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/encoding/base_encoder.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/encoding/base_encoder.py index 9829cadcd..eadc0a188 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/encoding/base_encoder.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/encoding/base_encoder.py @@ -28,5 +28,5 @@ def transform(self, X: Dict[str, Any]) -> Dict[str, Any]: if self.preprocessor['numerical'] is None and self.preprocessor['categorical'] is None: raise ValueError("cant call transform on {} without fitting first." .format(self.__class__.__name__)) - # X.update({'encoder': self.preprocessor}) + X.update({'encoder': self.preprocessor}) return X diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/imputation/base_imputer.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/imputation/base_imputer.py index 9bab21122..1f33a765a 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/imputation/base_imputer.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/imputation/base_imputer.py @@ -28,5 +28,5 @@ def transform(self, X: Dict[str, Any]) -> Dict[str, Any]: if self.preprocessor['numerical'] is None and len(X["dataset_properties"]["numerical_columns"]) != 0: raise ValueError("cant call transform on {} without fitting first." .format(self.__class__.__name__)) - # X.update({'imputer': self.preprocessor}) + X.update({'imputer': self.preprocessor}) return X diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/NoScaler.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/NoScaler.py index 9775d17dd..9d50aa8f5 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/NoScaler.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/NoScaler.py @@ -43,7 +43,7 @@ def transform(self, X: Dict[str, Any]) -> Dict[str, Any]: Returns: np.ndarray: Transformed features """ - # X.update({'scaler': self.preprocessor}) + X.update({'scaler': self.preprocessor}) return X @staticmethod diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/base_scaler.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/base_scaler.py index 270fac246..39834dd2b 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/base_scaler.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/base_scaler.py @@ -28,5 +28,5 @@ def transform(self, X: Dict[str, Any]) -> Dict[str, Any]: if self.preprocessor['numerical'] is None and self.preprocessor['categorical'] is None: raise ValueError("cant call transform on {} without fitting first." .format(self.__class__.__name__)) - # X.update({'scaler': self.preprocessor}) + X.update({'scaler': self.preprocessor}) return X diff --git a/autoPyTorch/pipeline/components/setup/early_preprocessor/EarlyPreprocessing.py b/autoPyTorch/pipeline/components/setup/early_preprocessor/EarlyPreprocessing.py index aa2b4c25f..597f14ca6 100644 --- a/autoPyTorch/pipeline/components/setup/early_preprocessor/EarlyPreprocessing.py +++ b/autoPyTorch/pipeline/components/setup/early_preprocessor/EarlyPreprocessing.py @@ -20,7 +20,6 @@ def __init__(self, random_state: Optional[np.random.RandomState] = None) -> None super().__init__() self.random_state = random_state self.add_fit_requirements([ - FitRequirement('is_small_preprocess', (bool,), user_defined=True, dataset_property=True), FitRequirement('X_train', (np.ndarray, pd.DataFrame, spmatrix), user_defined=True, dataset_property=False)]) @@ -32,14 +31,13 @@ def fit(self, X: Dict[str, Any], y: Any = None) -> "EarlyPreprocessing": def transform(self, X: Dict[str, Any]) -> Dict[str, Any]: transforms = get_preprocess_transforms(X) - if X['dataset_properties']['is_small_preprocess']: - if 'X_train' in X: - X_train = X['X_train'] - else: - # Incorporate the transform to the dataset - X_train = X['backend'].load_datamanager().train_tensors[0] - - X['X_train'] = preprocess(dataset=X_train, transforms=transforms) + if 'X_train' in X: + X_train = X['X_train'] + else: + # Incorporate the transform to the dataset + X_train = X['backend'].load_datamanager().train_tensors[0] + + X['X_train'] = preprocess(dataset=X_train, transforms=transforms) # We need to also save the preprocess transforms for inference X.update({'preprocess_transforms': transforms}) diff --git a/autoPyTorch/pipeline/components/setup/lr_scheduler/base_scheduler.py b/autoPyTorch/pipeline/components/setup/lr_scheduler/base_scheduler.py index 671a70f6a..bc53e2e1f 100644 --- a/autoPyTorch/pipeline/components/setup/lr_scheduler/base_scheduler.py +++ b/autoPyTorch/pipeline/components/setup/lr_scheduler/base_scheduler.py @@ -46,7 +46,7 @@ def transform(self, X: Dict[str, Any]) -> Dict[str, Any]: X.update( lr_scheduler=self.scheduler, step_interval=self.step_interval, - is_cyclic_scheduler= self.get_properties()['cyclic'] + is_cyclic_scheduler=self.get_properties()['cyclic'] ) return X diff --git a/autoPyTorch/pipeline/components/setup/network/base_network.py b/autoPyTorch/pipeline/components/setup/network/base_network.py index 02782e7a2..7ec872b96 100644 --- a/autoPyTorch/pipeline/components/setup/network/base_network.py +++ b/autoPyTorch/pipeline/components/setup/network/base_network.py @@ -1,5 +1,4 @@ -from typing import Any, Dict, Optional, Union -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Union from ConfigSpace.configuration_space import ConfigurationSpace diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/base_network_backbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/base_network_backbone.py index 7ff914a98..ef3cc1768 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/base_network_backbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/base_network_backbone.py @@ -28,7 +28,6 @@ def __init__(self, **kwargs: Any): super().__init__() self.add_fit_requirements([ - FitRequirement('is_small_preprocess', (bool,), user_defined=True, dataset_property=True), FitRequirement('X_train', (np.ndarray, pd.DataFrame, spmatrix), user_defined=True, dataset_property=False), FitRequirement('input_shape', (Iterable,), user_defined=True, dataset_property=True), @@ -52,12 +51,7 @@ def fit(self, X: Dict[str, Any], y: Any = None) -> BaseEstimator: self.check_requirements(X, y) X_train = X['X_train'] - if X["dataset_properties"]["is_small_preprocess"]: - input_shape = X_train.shape[1:] - else: - # get input shape by transforming first two elements of the training set - column_transformer = X['tabular_transformer'].preprocessor - input_shape = column_transformer.transform(X_train[:1]).shape[1:] + input_shape = X_train.shape[1:] input_shape = get_output_shape(X['network_embedding'], input_shape=input_shape) self.input_shape = input_shape diff --git a/autoPyTorch/pipeline/components/setup/network_embedding/base_network_embedding.py b/autoPyTorch/pipeline/components/setup/network_embedding/base_network_embedding.py index 334677f49..b99f253f3 100644 --- a/autoPyTorch/pipeline/components/setup/network_embedding/base_network_embedding.py +++ b/autoPyTorch/pipeline/components/setup/network_embedding/base_network_embedding.py @@ -1,7 +1,6 @@ import copy from typing import Any, Dict, List, Optional, Tuple, Union - import numpy as np from sklearn.base import BaseEstimator @@ -19,7 +18,7 @@ def __init__(self, random_state: Optional[np.random.RandomState] = None): def fit(self, X: Dict[str, Any], y: Any = None) -> BaseEstimator: - num_numerical_columns, num_input_features = self._get_args(X) + num_numerical_columns, num_input_features = self._get_required_info_from_data(X) self.embedding, num_output_features = self.build_embedding( num_input_features=num_input_features, @@ -50,31 +49,39 @@ def build_embedding(self, num_numerical_features: int) -> Tuple[nn.Module, Optional[List[int]]]: raise NotImplementedError - def _get_args(self, X: Dict[str, Any]) -> Tuple[None, None]: # Tuple[int, np.ndarray]: + def _get_required_info_from_data(self, X: Dict[str, Any]) -> Tuple[int, np.ndarray]: + """ + Returns the number of numerical columns after preprocessing and + an array of size equal to the number of input features + containing zeros for numerical data and number of categories + for categorical data. This is required to build the embedding. + + Args: + X (Dict[str, Any]): + Fit dictionary + + Returns: + Tuple[int, np.ndarray]: + number of numerical columns and array indicating + number of categories for categorical columns and + 0 for numerical columns + """ # Feature preprocessors can alter numerical columns if len(X['dataset_properties']['numerical_columns']) == 0: num_numerical_columns = 0 else: X_train = copy.deepcopy(X['backend'].load_datamanager().train_tensors[0][:2]) - if 'tabular_transformer' in X: - numerical_column_transformer = X['tabular_transformer'].preprocessor. \ - named_transformers_['numerical_pipeline'] - elif 'time_series_feature_transformer' in X: - numerical_column_transformer = X['time_series_feature_transformer'].preprocessor. \ - named_transformers_['numerical_pipeline'] - else: - raise ValueError("Either a tabular or time_series transformer must be contained!") - if hasattr(X_train, 'iloc'): - num_numerical_columns = numerical_column_transformer.transform( - X_train.iloc[:, X['dataset_properties']['numerical_columns']]).shape[1] - else: - num_numerical_columns = numerical_column_transformer.transform( - X_train[:, X['dataset_properties']['numerical_columns']]).shape[1] - num_input_features = np.zeros((num_numerical_columns + len(X['dataset_properties']['categorical_columns'])), - dtype=np.int32) + numerical_column_transformer = X['tabular_transformer'].preprocessor. \ + named_transformers_['numerical_pipeline'] + num_numerical_columns = numerical_column_transformer.transform( + X_train[:, X['dataset_properties']['numerical_columns']]).shape[1] + + num_cols = num_numerical_columns + len(X['dataset_properties']['categorical_columns']) + num_input_feats = np.zeros(num_cols, dtype=np.int32) + categories = X['dataset_properties']['categories'] + for idx, cats in enumerate(categories, start=num_numerical_columns): + num_input_feats[idx] = len(cats) - for i, category in enumerate(categories): - num_input_features[num_numerical_columns + i, ] = len(category) - return num_numerical_columns, num_input_features + return num_numerical_columns, num_input_feats diff --git a/autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py b/autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py index d99ba055c..3fb551adc 100644 --- a/autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py +++ b/autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py @@ -56,8 +56,8 @@ def __init__(self, batch_size: int = 64, # Define fit requirements self.add_fit_requirements([ FitRequirement("split_id", (int,), user_defined=True, dataset_property=False), - FitRequirement("Backend", (Backend,), user_defined=True, dataset_property=False), - FitRequirement("is_small_preprocess", (bool,), user_defined=True, dataset_property=True)]) + FitRequirement("Backend", (Backend,), user_defined=True, dataset_property=False) + ]) def transform(self, X: Dict[str, Any]) -> Dict[str, Any]: """The transform function calls the transform function of the @@ -102,10 +102,9 @@ def fit(self, X: Dict[str, Any], y: Any = None) -> torch.utils.data.DataLoader: self.val_transform, train=False, ) - if X['dataset_properties']["is_small_preprocess"]: - # This parameter indicates that the data has been pre-processed for speed - # Overwrite the datamanager with the pre-processes data - datamanager.replace_data(X['X_train'], X['X_test'] if 'X_test' in X else None) + # This parameter indicates that the data has been pre-processed for speed + # Overwrite the datamanager with the pre-processes data + datamanager.replace_data(X['X_train'], X['X_test'] if 'X_test' in X else None) train_dataset = datamanager.get_dataset(split_id=X['split_id'], train=True) @@ -221,10 +220,6 @@ def check_requirements(self, X: Dict[str, Any], y: Any = None) -> None: if 'backend' not in X: raise ValueError("backend is needed to load the data from disk") - if 'is_small_preprocess' not in X['dataset_properties']: - raise ValueError("is_small_pre-process is required to know if the data was preprocessed" - " or if the data-loader should transform it while loading a batch") - # We expect this class to be a base for image/tabular/time # And the difference among this data types should be mainly # in the transform, so we delegate for special transformation checking diff --git a/autoPyTorch/pipeline/components/training/data_loader/feature_data_loader.py b/autoPyTorch/pipeline/components/training/data_loader/feature_data_loader.py index 4e41ec838..d6f3081a0 100644 --- a/autoPyTorch/pipeline/components/training/data_loader/feature_data_loader.py +++ b/autoPyTorch/pipeline/components/training/data_loader/feature_data_loader.py @@ -72,7 +72,7 @@ def build_transform(self, X: Dict[str, Any], mode: str) -> torchvision.transform # distinction is performed candidate_transformations: List[Callable] = [] - if 'test' in mode or not X['dataset_properties']['is_small_preprocess']: + if 'test' in mode: candidate_transformations.append((ExpandTransform())) candidate_transformations.extend(X['preprocess_transforms']) candidate_transformations.append((ContractTransform())) @@ -93,5 +93,5 @@ def _check_transform_requirements(self, X: Dict[str, Any], y: Any = None) -> Non mechanism, in which during a transform, a components adds relevant information so that further stages can be properly fitted """ - if not X['dataset_properties']['is_small_preprocess'] and 'preprocess_transforms' not in X: + if 'preprocess_transforms' not in X: raise ValueError("Cannot find the preprocess_transforms in the fit dictionary") diff --git a/autoPyTorch/pipeline/components/training/data_loader/image_data_loader.py b/autoPyTorch/pipeline/components/training/data_loader/image_data_loader.py index 21cc05447..38cdd48b0 100644 --- a/autoPyTorch/pipeline/components/training/data_loader/image_data_loader.py +++ b/autoPyTorch/pipeline/components/training/data_loader/image_data_loader.py @@ -41,7 +41,7 @@ def build_transform(self, X: Dict[str, Any], mode: str) -> torchvision.transform # check if data set is small enough to be preprocessed. # If it is, then no need to add preprocess_transforms to # the data loader as the data is already preprocessed - if 'test' in mode or not X['dataset_properties']['is_small_preprocess']: + if 'test' in mode: transformations.append(X['preprocess_transforms']) # Transform to tensor @@ -63,5 +63,5 @@ def _check_transform_requirements(self, X: Dict[str, Any], y: Any = None) -> Non if not X['image_augmenter'] and 'image_augmenter' not in X: raise ValueError("Cannot find the image_augmenter in the fit dictionary") - if not X['dataset_properties']['is_small_preprocess'] and 'preprocess_transforms' not in X: + if 'preprocess_transforms' not in X: raise ValueError("Cannot find the preprocess_transforms in the fit dictionary") diff --git a/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py b/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py index 67ae71188..fc78e4655 100644 --- a/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py @@ -76,7 +76,7 @@ def criterion_preparation(self, y_a: np.ndarray, y_b: np.ndarray = None, lam: fl # Initial implementation, consider the adversarial loss and the normal network loss # equally. return lambda criterion, pred, adversarial_pred: 0.5 * criterion(pred, y_a) + \ - 0.5 * criterion(adversarial_pred, y_a) + 0.5 * criterion(adversarial_pred, y_a) def train_step(self, data: np.ndarray, targets: np.ndarray) -> Tuple[float, torch.Tensor]: """ diff --git a/autoPyTorch/pipeline/components/training/trainer/__init__.py b/autoPyTorch/pipeline/components/training/trainer/__init__.py index 144f703f7..e6a7acfb4 100755 --- a/autoPyTorch/pipeline/components/training/trainer/__init__.py +++ b/autoPyTorch/pipeline/components/training/trainer/__init__.py @@ -86,7 +86,7 @@ def get_fit_requirements(self) -> Optional[List[FitRequirement]]: def get_available_components( self, - dataset_properties: Optional[Dict[str, str]] = None, + dataset_properties: Optional[Dict[str, BaseDatasetPropertiesType]] = None, include: Optional[List[str]] = None, exclude: Optional[List[str]] = None, ) -> Dict[str, autoPyTorchComponent]: diff --git a/autoPyTorch/pipeline/components/training/trainer/base_trainer.py b/autoPyTorch/pipeline/components/training/trainer/base_trainer.py index 7daca56d0..a6602a88f 100644 --- a/autoPyTorch/pipeline/components/training/trainer/base_trainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/base_trainer.py @@ -29,7 +29,6 @@ FORECASTING_METRICS, REGRESSION_METRICS, ) -from autoPyTorch.pipeline.components.training.trainer.utils import Lookahead from autoPyTorch.pipeline.components.training.metrics.utils import calculate_score from autoPyTorch.pipeline.components.training.trainer.utils import Lookahead, swa_update from autoPyTorch.utils.common import FitRequirement, HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter diff --git a/test/test_data/test_feature_validator.py b/test/test_data/test_feature_validator.py index 499ba37c3..293d48de6 100644 --- a/test/test_data/test_feature_validator.py +++ b/test/test_data/test_feature_validator.py @@ -167,10 +167,120 @@ def test_featurevalidator_get_columns_to_encode(): for col in df.columns: df[col] = df[col].astype(col) - transformed_columns, feature_types = validator._get_columns_to_encode(df) + validator.fit(df) - assert transformed_columns == ['category', 'bool'] - assert feature_types == ['numerical', 'numerical', 'categorical', 'categorical'] + categorical_columns, feat_type = validator._get_columns_info(df) + + assert categorical_columns == ['category', 'bool'] + assert feat_type == ['numerical', 'numerical', 'categorical', 'categorical'] + + +def feature_validator_remove_nan_catcolumns(df_train: pd.DataFrame, df_test: pd.DataFrame, + ans_train: np.ndarray, ans_test: np.ndarray) -> None: + validator = TabularFeatureValidator() + validator.fit(df_train) + transformed_df_train = validator.transform(df_train) + transformed_df_test = validator.transform(df_test) + + np.testing.assert_array_equal(transformed_df_train, ans_train) + np.testing.assert_array_equal(transformed_df_test, ans_test) + + +def test_feature_validator_remove_nan_catcolumns(): + """ + Make sure categorical columns that have only nan values are removed. + Transform performs the folloing: + * simple imputation for both + * scaling for numerical + * one-hot encoding for categorical + For example, + data = [ + {'A': 1, 'B': np.nan, 'C': np.nan}, + {'A': np.nan, 'B': 3, 'C': np.nan}, + {'A': 2, 'B': np.nan, 'C': np.nan} + ] + and suppose all the columns are categorical, + then + * `A` in {np.nan, 1, 2} + * `B` in {np.nan, 3} + * `C` in {np.nan} <=== it will be dropped. + + So in the column A, + * np.nan ==> [1, 0, 0] + * 1 ==> [0, 1, 0] + * 2 ==> [0, 0, 1] + in the column B, + * np.nan ==> [1, 0] + * 3 ==> [0, 1] + Therefore, by concatenating, + * {'A': 1, 'B': np.nan, 'C': np.nan} ==> [0, 1, 0, 1, 0] + * {'A': np.nan, 'B': 3, 'C': np.nan} ==> [1, 0, 0, 0, 1] + * {'A': 2, 'B': np.nan, 'C': np.nan} ==> [0, 0, 1, 1, 0] + """ + # First case, there exist null columns (B and C) in the train set + # and a same column (C) are not all null for the test set. + + df_train = pd.DataFrame( + [ + {'A': 1, 'B': np.nan, 'C': np.nan}, + {'A': np.nan, 'C': np.nan}, + {'A': 1} + ], + dtype='category', + ) + ans_train = np.array([[1, np.nan, np.nan], [0, np.nan, np.nan], [1, np.nan, np.nan]], dtype=np.float64) + df_test = pd.DataFrame( + [ + {'A': np.nan, 'B': np.nan, 'C': 5}, + {'A': np.nan, 'C': np.nan}, + {'A': 1} + ], + dtype='category', + ) + ans_test = np.array([[0, np.nan, np.nan], [0, np.nan, np.nan], [1, np.nan, np.nan]], dtype=np.float64) + feature_validator_remove_nan_catcolumns(df_train, df_test, ans_train, ans_test) + + # Second case, there exist null columns (B and C) in the training set and + # the same columns (B and C) are null in the test set. + df_train = pd.DataFrame( + [ + {'A': 1, 'B': np.nan, 'C': np.nan}, + {'A': np.nan, 'C': np.nan}, + {'A': 1} + ], + dtype='category', + ) + ans_train = np.array([[1, np.nan, np.nan], [0, np.nan, np.nan], [1, np.nan, np.nan]], dtype=np.float64) + df_test = pd.DataFrame( + [ + {'A': np.nan, 'B': np.nan, 'C': np.nan}, + {'A': np.nan, 'C': np.nan}, + {'A': 1} + ], + dtype='category', + ) + ans_test = np.array([[0, np.nan, np.nan], [0, np.nan, np.nan], [1, np.nan, np.nan]], dtype=np.float64) + feature_validator_remove_nan_catcolumns(df_train, df_test, ans_train, ans_test) + + # Third case, there exist no null columns in the training set and + # null columns exist in the test set. + df_train = pd.DataFrame( + [ + {'A': 1, 'B': 1}, + {'A': 2, 'B': 2} + ], + dtype='category', + ) + ans_train = np.array([[0, 0], [1, 1]], dtype=np.float64) + df_test = pd.DataFrame( + [ + {'A': np.nan, 'B': np.nan}, + {'A': np.nan, 'B': np.nan} + ], + dtype='category', + ) + ans_test = np.array([[-1, -1], [-1, -1]], dtype=np.float64) + feature_validator_remove_nan_catcolumns(df_train, df_test, ans_train, ans_test) def test_features_unsupported_calls_are_raised(): @@ -256,15 +366,15 @@ def test_column_transformer_created(input_data_featuretest): # Make sure that the encoded features are actually encoded. Categorical columns are at # the start after transformation. In our fixtures, this is also honored prior encode - transformed_columns, feature_types = validator._get_columns_to_encode(input_data_featuretest) + cat_columns, feature_types = validator._get_columns_info(input_data_featuretest) # At least one categorical - assert 'categorical' in validator.feat_types + assert 'categorical' in validator.feat_typess # Numerical if the original data has numerical only columns if np.any([pd.api.types.is_numeric_dtype(input_data_featuretest[col] ) for col in input_data_featuretest.columns]): - assert 'numerical' in validator.feat_types + assert 'numerical' in validator.feat_typess for i, feat_type in enumerate(feature_types): if 'numerical' in feat_type: np.testing.assert_array_equal( @@ -551,7 +661,7 @@ def test_feature_validator_imbalanced_data(): validator = TabularFeatureValidator() validator.fit(X_train) - train_feature_types = copy.deepcopy(validator.feat_type) + train_feature_types = copy.deepcopy(validator.feat_types) assert train_feature_types == ['numerical', 'numerical', 'numerical', 'numerical'] # validator will throw an error if the column types are not the same transformed_X_test = validator.transform(X_test) @@ -561,6 +671,7 @@ def test_feature_validator_imbalanced_data(): if transformed_X_test[column].isna().all(): null_columns.append(column) assert null_columns == [0, 2, 3] + assert sorted(validator.all_nan_columns) == sorted(['A', 'C', 'D']) # Columns with not all null values in the train split and # completely null on the test split. @@ -579,7 +690,7 @@ def test_feature_validator_imbalanced_data(): X_test = pd.DataFrame.from_dict(test_features) validator = TabularFeatureValidator() validator.fit(X_train) - train_feature_types = copy.deepcopy(validator.feat_type) + train_feature_types = copy.deepcopy(validator.feat_types) assert train_feature_types == ['categorical', 'numerical', 'numerical'] transformed_X_test = validator.transform(X_test) diff --git a/test/test_data/test_validation.py b/test/test_data/test_validation.py index b58b29b59..af46be55f 100644 --- a/test/test_data/test_validation.py +++ b/test/test_data/test_validation.py @@ -85,7 +85,7 @@ def test_sparse_data_validation_for_regression(): validator.fit(X_train=X_sp, y_train=y) - X_t, y_t = validator.transform(X, y) + X_t, y_t = validator.transform(X_sp, y) # make sure everything was encoded to number assert np.issubdtype(X_t.dtype, np.number) assert np.issubdtype(y_t.dtype, np.number) diff --git a/test/test_datasets/test_tabular_dataset.py b/test/test_datasets/test_tabular_dataset.py index 2ee8b608e..710111f9c 100644 --- a/test/test_datasets/test_tabular_dataset.py +++ b/test/test_datasets/test_tabular_dataset.py @@ -28,7 +28,6 @@ def test_get_dataset_properties(backend, fit_dictionary_tabular): 'categorical_columns', 'numerical_columns', 'issparse', - 'is_small_preprocess', 'task_type', 'output_type', 'input_shape', diff --git a/test/test_evaluation/test_fit_evaluator.py b/test/test_evaluation/test_fit_evaluator.py deleted file mode 100644 index 1515ba74f..000000000 --- a/test/test_evaluation/test_fit_evaluator.py +++ /dev/null @@ -1,206 +0,0 @@ -import multiprocessing -import os -import queue -import shutil -import sys -import unittest -import unittest.mock - -from ConfigSpace import Configuration - -import numpy as np - -from sklearn.base import BaseEstimator - -from smac.tae import StatusType - -from autoPyTorch.automl_common.common.utils.backend import create -from autoPyTorch.datasets.resampling_strategy import NoResamplingStrategyTypes -from autoPyTorch.evaluation.fit_evaluator import FitEvaluator -from autoPyTorch.evaluation.utils import read_queue -from autoPyTorch.pipeline.base_pipeline import BasePipeline -from autoPyTorch.pipeline.components.training.metrics.metrics import accuracy - -this_directory = os.path.dirname(__file__) -sys.path.append(this_directory) -from evaluation_util import ( # noqa (E402: module level import not at top of file) - BaseEvaluatorTest, - get_binary_classification_datamanager, - get_multiclass_classification_datamanager, - get_regression_datamanager, -) # noqa (E402: module level import not at top of file) - - -class BackendMock(object): - def load_datamanager(self): - return get_multiclass_classification_datamanager() - - -class Dummy(object): - def __init__(self): - self.name = 'dummy' - - -class DummyPipeline(BasePipeline): - def __init__(self): - mocked_estimator = unittest.mock.Mock(spec=BaseEstimator) - self.steps = [('MockStep', mocked_estimator)] - pass - - def predict_proba(self, X, batch_size=None): - return np.tile([0.6, 0.4], (len(X), 1)) - - def get_additional_run_info(self): - return {} - - -class TestFitEvaluator(BaseEvaluatorTest, unittest.TestCase): - _multiprocess_can_split_ = True - - def setUp(self): - """ - Creates a backend mock - """ - tmp_dir_name = self.id() - self.ev_path = os.path.join(this_directory, '.tmp_evaluations', tmp_dir_name) - if os.path.exists(self.ev_path): - shutil.rmtree(self.ev_path) - os.makedirs(self.ev_path, exist_ok=False) - dummy_model_files = [os.path.join(self.ev_path, str(n)) for n in range(100)] - dummy_pred_files = [os.path.join(self.ev_path, str(n)) for n in range(100, 200)] - dummy_cv_model_files = [os.path.join(self.ev_path, str(n)) for n in range(200, 300)] - backend_mock = unittest.mock.Mock() - backend_mock.get_model_dir.return_value = self.ev_path - backend_mock.get_cv_model_dir.return_value = self.ev_path - backend_mock.get_model_path.side_effect = dummy_model_files - backend_mock.get_cv_model_path.side_effect = dummy_cv_model_files - backend_mock.get_prediction_output_path.side_effect = dummy_pred_files - backend_mock.temporary_directory = self.ev_path - self.backend_mock = backend_mock - - self.tmp_dir = os.path.join(self.ev_path, 'tmp_dir') - self.output_dir = os.path.join(self.ev_path, 'out_dir') - - def tearDown(self): - if os.path.exists(self.ev_path): - shutil.rmtree(self.ev_path) - - @unittest.mock.patch('autoPyTorch.pipeline.tabular_classification.TabularClassificationPipeline') - def test_no_resampling(self, pipeline_mock): - # Binary iris, contains 69 train samples, 31 test samples - D = get_binary_classification_datamanager(NoResamplingStrategyTypes.no_resampling) - pipeline_mock.predict_proba.side_effect = \ - lambda X, batch_size=None: np.tile([0.6, 0.4], (len(X), 1)) - pipeline_mock.side_effect = lambda **kwargs: pipeline_mock - pipeline_mock.get_additional_run_info.return_value = None - pipeline_mock.get_default_pipeline_options.return_value = {'budget_type': 'epochs', 'epochs': 10} - - configuration = unittest.mock.Mock(spec=Configuration) - backend_api = create(self.tmp_dir, self.output_dir, 'autoPyTorch') - backend_api.load_datamanager = lambda: D - queue_ = multiprocessing.Queue() - - evaluator = FitEvaluator(backend_api, queue_, configuration=configuration, metric=accuracy, budget=0) - evaluator.file_output = unittest.mock.Mock(spec=evaluator.file_output) - evaluator.file_output.return_value = (None, {}) - - evaluator.fit_predict_and_loss() - - rval = read_queue(evaluator.queue) - self.assertEqual(len(rval), 1) - result = rval[0]['loss'] - self.assertEqual(len(rval[0]), 3) - self.assertRaises(queue.Empty, evaluator.queue.get, timeout=1) - - self.assertEqual(evaluator.file_output.call_count, 1) - self.assertEqual(result, 0.5806451612903225) - self.assertEqual(pipeline_mock.fit.call_count, 1) - # 2 calls because of train and test set - self.assertEqual(pipeline_mock.predict_proba.call_count, 2) - self.assertEqual(evaluator.file_output.call_count, 1) - # Should be none as no val preds are mentioned - self.assertIsNone(evaluator.file_output.call_args[0][0]) - # Number of y_test_preds and Y_test should be the same - self.assertEqual(evaluator.file_output.call_args[0][2].shape[0], - D.test_tensors[1].shape[0]) - self.assertEqual(evaluator.pipeline.fit.call_count, 1) - - @unittest.mock.patch.object(FitEvaluator, '_loss') - def test_file_output(self, loss_mock): - - D = get_regression_datamanager(NoResamplingStrategyTypes.no_resampling) - D.name = 'test' - self.backend_mock.load_datamanager.return_value = D - configuration = unittest.mock.Mock(spec=Configuration) - queue_ = multiprocessing.Queue() - loss_mock.return_value = None - - evaluator = FitEvaluator(self.backend_mock, queue_, configuration=configuration, metric=accuracy, budget=0) - - self.backend_mock.get_model_dir.return_value = True - evaluator.pipeline = 'model' - evaluator.Y_optimization = D.train_tensors[1] - rval = evaluator.file_output( - D.train_tensors[1], - None, - D.test_tensors[1], - ) - - self.assertEqual(rval, (None, {})) - # These targets are not saved as Fit evaluator is not used to make an ensemble - self.assertEqual(self.backend_mock.save_targets_ensemble.call_count, 0) - self.assertEqual(self.backend_mock.save_numrun_to_dir.call_count, 1) - self.assertEqual(self.backend_mock.save_numrun_to_dir.call_args_list[-1][1].keys(), - {'seed', 'idx', 'budget', 'model', 'cv_model', - 'ensemble_predictions', 'valid_predictions', 'test_predictions'}) - self.assertIsNotNone(self.backend_mock.save_numrun_to_dir.call_args_list[-1][1]['model']) - self.assertIsNone(self.backend_mock.save_numrun_to_dir.call_args_list[-1][1]['cv_model']) - - # Check for not containing NaNs - that the models don't predict nonsense - # for unseen data - D.test_tensors[1][0] = np.NaN - rval = evaluator.file_output( - D.train_tensors[1], - None, - D.test_tensors[1], - ) - self.assertEqual( - rval, - ( - 1.0, - { - 'error': - 'Model predictions for test set contains NaNs.' - }, - ) - ) - - @unittest.mock.patch('autoPyTorch.pipeline.tabular_classification.TabularClassificationPipeline') - def test_predict_proba_binary_classification(self, mock): - D = get_binary_classification_datamanager(NoResamplingStrategyTypes.no_resampling) - self.backend_mock.load_datamanager.return_value = D - mock.predict_proba.side_effect = lambda y, batch_size=None: np.array( - [[0.1, 0.9]] * y.shape[0] - ) - mock.side_effect = lambda **kwargs: mock - mock.get_default_pipeline_options.return_value = {'budget_type': 'epochs', 'epochs': 10} - configuration = unittest.mock.Mock(spec=Configuration) - queue_ = multiprocessing.Queue() - - evaluator = FitEvaluator(self.backend_mock, queue_, configuration=configuration, metric=accuracy, budget=0) - - evaluator.fit_predict_and_loss() - Y_test_pred = self.backend_mock.save_numrun_to_dir.call_args_list[0][1][ - 'test_predictions'] - - for i in range(7): - self.assertEqual(0.9, Y_test_pred[i][1]) - - def test_get_results(self): - queue_ = multiprocessing.Queue() - for i in range(5): - queue_.put((i * 1, 1 - (i * 0.2), 0, "", StatusType.SUCCESS)) - result = read_queue(queue_) - self.assertEqual(len(result), 5) - self.assertEqual(result[0][0], 0) - self.assertAlmostEqual(result[0][1], 1.0) diff --git a/test/test_pipeline/components/preprocessing/test_encoders.py b/test/test_pipeline/components/preprocessing/test_encoders.py index ac796291c..a901823ba 100644 --- a/test/test_pipeline/components/preprocessing/test_encoders.py +++ b/test/test_pipeline/components/preprocessing/test_encoders.py @@ -10,8 +10,6 @@ from autoPyTorch.pipeline.components.preprocessing.tabular_preprocessing.encoding.OneHotEncoder import OneHotEncoder -# TODO: fix in preprocessing PR -@unittest.skip("Skipping tests as preprocessing is not finalised") class TestEncoders(unittest.TestCase): def test_one_hot_encoder_no_unknown(self): diff --git a/test/test_pipeline/components/preprocessing/test_imputers.py b/test/test_pipeline/components/preprocessing/test_imputers.py index ad9ed710f..0db460b77 100644 --- a/test/test_pipeline/components/preprocessing/test_imputers.py +++ b/test/test_pipeline/components/preprocessing/test_imputers.py @@ -11,8 +11,6 @@ from autoPyTorch.pipeline.components.preprocessing.tabular_preprocessing.imputation.SimpleImputer import SimpleImputer -# TODO: fix in preprocessing PR -@unittest.skip("Skipping tests as preprocessing is not finalised") class TestSimpleImputer(unittest.TestCase): def test_get_config_space(self): diff --git a/test/test_pipeline/components/preprocessing/test_scalers.py b/test/test_pipeline/components/preprocessing/test_scalers.py index 8d05c8da1..7cbc12b07 100644 --- a/test/test_pipeline/components/preprocessing/test_scalers.py +++ b/test/test_pipeline/components/preprocessing/test_scalers.py @@ -17,8 +17,6 @@ from autoPyTorch.pipeline.components.preprocessing.tabular_preprocessing.scaling.StandardScaler import StandardScaler -# TODO: fix in preprocessing PR -@unittest.skip("Skipping tests as preprocessing is not finalised") class TestNormalizer(unittest.TestCase): def test_l2_norm(self): @@ -136,8 +134,6 @@ def test_max_norm(self): [0.84615385, 0.92307692, 1]])) -# TODO: fix in preprocessing PR -@unittest.skip("Skipping tests as preprocessing is not finalised") class TestMinMaxScaler(unittest.TestCase): def test_minmax_scaler(self): @@ -179,8 +175,6 @@ def test_minmax_scaler(self): [0.76923077, 0.76923077, 0.76923077]])) -# TODO: fix in preprocessing PR -@unittest.skip("Skipping tests as preprocessing is not finalised") class TestStandardScaler(unittest.TestCase): def test_standard_scaler(self): @@ -223,8 +217,6 @@ def test_standard_scaler(self): [0.8396642, 0.8396642, 0.8396642]])) -# TODO: fix in preprocessing PR -@unittest.skip("Skipping tests as preprocessing is not finalised") class TestNoneScaler(unittest.TestCase): def test_none_scaler(self): diff --git a/test/test_pipeline/components/preprocessing/test_tabular_column_transformer.py b/test/test_pipeline/components/preprocessing/test_tabular_column_transformer.py index 6db124be1..a81eb34a2 100644 --- a/test/test_pipeline/components/preprocessing/test_tabular_column_transformer.py +++ b/test/test_pipeline/components/preprocessing/test_tabular_column_transformer.py @@ -14,13 +14,14 @@ # TODO: fix in preprocessing PR -@pytest.mark.skip("Skipping tests as preprocessing is not finalised") +# @pytest.mark.skip("Skipping tests as preprocessing is not finalised") @pytest.mark.parametrize("fit_dictionary_tabular", ['classification_numerical_only', 'classification_categorical_only', 'classification_numerical_and_categorical'], indirect=True) class TestTabularTransformer: def test_tabular_preprocess(self, fit_dictionary_tabular): pipeline = TabularPipeline(dataset_properties=fit_dictionary_tabular['dataset_properties']) + X_train = fit_dictionary_tabular['X_train'].copy() pipeline = pipeline.fit(fit_dictionary_tabular) X = pipeline.transform(fit_dictionary_tabular) column_transformer = X['tabular_transformer'] @@ -32,17 +33,17 @@ def test_tabular_preprocess(self, fit_dictionary_tabular): # as the later is not callable and runs into error in the compose transform assert isinstance(column_transformer, TabularColumnTransformer) - data = column_transformer.preprocessor.fit_transform(X['X_train']) + data = column_transformer.preprocessor.fit_transform(X_train) assert isinstance(data, np.ndarray) # Make sure no columns are unintentionally dropped after preprocessing if len(fit_dictionary_tabular['dataset_properties']["numerical_columns"]) == 0: categorical_pipeline = column_transformer.preprocessor.named_transformers_['categorical_pipeline'] - categorical_data = categorical_pipeline.transform(X['X_train']) + categorical_data = categorical_pipeline.transform(X_train) assert data.shape[1] == categorical_data.shape[1] elif len(fit_dictionary_tabular['dataset_properties']["categorical_columns"]) == 0: numerical_pipeline = column_transformer.preprocessor.named_transformers_['numerical_pipeline'] - numerical_data = numerical_pipeline.transform(X['X_train']) + numerical_data = numerical_pipeline.transform(X_train) assert data.shape[1] == numerical_data.shape[1] def test_sparse_data(self, fit_dictionary_tabular): diff --git a/test/test_pipeline/components/training/test_feature_data_loader.py b/test/test_pipeline/components/training/test_feature_data_loader.py index 7d4c9d80d..7e97494a4 100644 --- a/test/test_pipeline/components/training/test_feature_data_loader.py +++ b/test/test_pipeline/components/training/test_feature_data_loader.py @@ -9,13 +9,13 @@ class TestFeatureDataLoader(unittest.TestCase): - def test_build_transform_small_preprocess_true(self): + def test_build_transform(self): """ Makes sure a proper composition is created """ loader = FeatureDataLoader() - fit_dictionary = {'dataset_properties': {'is_small_preprocess': True}} + fit_dictionary = {'dataset_properties': {}} for thing in ['imputer', 'scaler', 'encoder']: fit_dictionary[thing] = [unittest.mock.Mock()] @@ -25,19 +25,3 @@ def test_build_transform_small_preprocess_true(self): # No preprocessing needed here as it was done before self.assertEqual(len(compose.transforms), 1) - - def test_build_transform_small_preprocess_false(self): - """ - Makes sure a proper composition is created - """ - loader = FeatureDataLoader() - - fit_dictionary = {'dataset_properties': {'is_small_preprocess': False}, - 'preprocess_transforms': [unittest.mock.Mock()]} - - compose = loader.build_transform(fit_dictionary, mode='train') - - self.assertIsInstance(compose, torchvision.transforms.Compose) - - # We expect the to tensor, the preproces transforms and the check_array - self.assertEqual(len(compose.transforms), 4) diff --git a/test/test_pipeline/components/training/test_training.py b/test/test_pipeline/components/training/test_training.py index 39fa7668e..c011cea38 100644 --- a/test/test_pipeline/components/training/test_training.py +++ b/test/test_pipeline/components/training/test_training.py @@ -93,12 +93,6 @@ def test_check_requirements(self): 'backend is needed to load the data from'): loader.fit(fit_dictionary) - # Then the is small fit - fit_dictionary.update({'backend': unittest.mock.Mock()}) - with self.assertRaisesRegex(ValueError, - 'is_small_pre-process is required to know if th'): - loader.fit(fit_dictionary) - def test_fit_transform(self): """ Makes sure that fit and transform work as intended """ backend = unittest.mock.Mock() From c1fffa1a01c5cc0d8c458199419f82916ecbfc7f Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Mon, 28 Feb 2022 14:59:26 +0100 Subject: [PATCH 42/50] fixes after rebase --- autoPyTorch/api/tabular_classification.py | 8 ++++---- autoPyTorch/api/tabular_regression.py | 6 +++--- autoPyTorch/data/base_feature_validator.py | 12 ++++++------ autoPyTorch/data/tabular_target_validator.py | 2 +- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/autoPyTorch/api/tabular_classification.py b/autoPyTorch/api/tabular_classification.py index 3ccfa88ea..facb59f99 100644 --- a/autoPyTorch/api/tabular_classification.py +++ b/autoPyTorch/api/tabular_classification.py @@ -486,23 +486,23 @@ def predict( raise ValueError("predict() is only supported after calling search. Kindly call first " "the estimator search() method.") - X_test = self.InputValidator.feature_validator.transform(X_test) + X_test = self.input_validator.feature_validator.transform(X_test) predicted_probabilities = super().predict(X_test, batch_size=batch_size, n_jobs=n_jobs) - if self.InputValidator.target_validator.is_single_column_target(): + if self.input_validator.target_validator.is_single_column_target(): predicted_indexes = np.argmax(predicted_probabilities, axis=1) else: predicted_indexes = (predicted_probabilities > 0.5).astype(int) # Allow to predict in the original domain -- that is, the user is not interested # in our encoded values - return self.InputValidator.target_validator.inverse_transform(predicted_indexes) + return self.input_validator.target_validator.inverse_transform(predicted_indexes) def predict_proba(self, X_test: Union[np.ndarray, pd.DataFrame, List], batch_size: Optional[int] = None, n_jobs: int = 1) -> np.ndarray: - if self.InputValidator is None or not self.InputValidator._is_fitted: + if self.input_validator is None or not self.input_validator._is_fitted: raise ValueError("predict() is only supported after calling search. Kindly call first " "the estimator search() method.") X_test = self.input_validator.feature_validator.transform(X_test) diff --git a/autoPyTorch/api/tabular_regression.py b/autoPyTorch/api/tabular_regression.py index d2b087ddb..faf36097a 100644 --- a/autoPyTorch/api/tabular_regression.py +++ b/autoPyTorch/api/tabular_regression.py @@ -472,14 +472,14 @@ def predict( batch_size: Optional[int] = None, n_jobs: int = 1 ) -> np.ndarray: - if self.InputValidator is None or not self.InputValidator._is_fitted: + if self.input_validator is None or not self.input_validator._is_fitted: raise ValueError("predict() is only supported after calling search. Kindly call first " "the estimator search() method.") - X_test = self.InputValidator.feature_validator.transform(X_test) + X_test = self.input_validator.feature_validator.transform(X_test) predicted_values = super().predict(X_test, batch_size=batch_size, n_jobs=n_jobs) # Allow to predict in the original domain -- that is, the user is not interested # in our encoded values - return self.InputValidator.target_validator.inverse_transform(predicted_values) + return self.input_validator.target_validator.inverse_transform(predicted_values) diff --git a/autoPyTorch/data/base_feature_validator.py b/autoPyTorch/data/base_feature_validator.py index 38d5e0ef7..0f2ba35ae 100644 --- a/autoPyTorch/data/base_feature_validator.py +++ b/autoPyTorch/data/base_feature_validator.py @@ -113,13 +113,13 @@ def _fit( def _check_data( self, - X: SUPPORTED_FEAT_TYPES, + X: SupportedFeatTypes, ) -> None: """ Feature dimensionality and data type checks Args: - X (SUPPORTED_FEAT_TYPES): + X (SupportedFeatTypes): A set of features that are going to be validated (type and dimensionality checks) and a encoder fitted in the case the data needs encoding """ @@ -145,8 +145,8 @@ def transform( def list_to_pandas( self, - X_train: SUPPORTED_FEAT_TYPES, - X_test: Optional[SUPPORTED_FEAT_TYPES] = None, + X_train: SupportedFeatTypes, + X_test: Optional[SupportedFeatTypes] = None, ) -> Tuple[pd.DataFrame, Optional[pd.DataFrame]]: """ Converts a list to a pandas DataFrame. In this process, column types are inferred. @@ -154,10 +154,10 @@ def list_to_pandas( If test data is provided, we proactively match it to train data Args: - X_train (SUPPORTED_FEAT_TYPES): + X_train (SupportedFeatTypes): A set of features that are going to be validated (type and dimensionality checks) and a encoder fitted in the case the data needs encoding - X_test (Optional[SUPPORTED_FEAT_TYPES]): + X_test (Optional[SupportedFeatTypes]): A hold out set of data used for checking Returns: pd.DataFrame: diff --git a/autoPyTorch/data/tabular_target_validator.py b/autoPyTorch/data/tabular_target_validator.py index 8f0c765f6..3f1aa2f96 100644 --- a/autoPyTorch/data/tabular_target_validator.py +++ b/autoPyTorch/data/tabular_target_validator.py @@ -1,4 +1,4 @@ -from typing import List, Optional, cast +from typing import List, Optional, Union, cast import numpy as np import numpy.ma as ma From 366beded0614a1de7134734ebde1710a405e51f9 Mon Sep 17 00:00:00 2001 From: Ravin Kohli <13005107+ravinkohli@users.noreply.github.com> Date: Wed, 9 Mar 2022 16:49:54 +0100 Subject: [PATCH 43/50] [FIX] SWA and SE with non cyclic schedulers (#395) * Enable learned embeddings, fix bug with non cyclic schedulers * add forbidden condition cyclic lr * refactor base_pipeline forbidden conditions * Apply suggestions from code review Co-authored-by: nabenabe0928 <47781922+nabenabe0928@users.noreply.github.com> Co-authored-by: nabenabe0928 <47781922+nabenabe0928@users.noreply.github.com> --- autoPyTorch/pipeline/base_pipeline.py | 63 +++++++++++ .../setup/network_embedding/__init__.py | 103 ++++++++---------- .../training/trainer/base_trainer.py | 71 ++++++------ autoPyTorch/pipeline/image_classification.py | 1 + .../pipeline/tabular_classification.py | 31 +----- autoPyTorch/pipeline/tabular_regression.py | 29 +---- 6 files changed, 156 insertions(+), 142 deletions(-) diff --git a/autoPyTorch/pipeline/base_pipeline.py b/autoPyTorch/pipeline/base_pipeline.py index 7c2efa798..e48f37510 100644 --- a/autoPyTorch/pipeline/base_pipeline.py +++ b/autoPyTorch/pipeline/base_pipeline.py @@ -1,3 +1,4 @@ +from copy import copy import warnings from abc import ABCMeta from collections import Counter @@ -5,6 +6,7 @@ from ConfigSpace import Configuration from ConfigSpace.configuration_space import ConfigurationSpace +from ConfigSpace.forbidden import ForbiddenAndConjunction, ForbiddenEqualsClause import numpy as np @@ -295,6 +297,67 @@ def _get_hyperparameter_search_space(self, """ raise NotImplementedError() + def _add_forbidden_conditions(self, cs): + """ + Add forbidden conditions to ensure valid configurations. + Currently, Learned Entity Embedding is only valid when encoder is one hot encoder + and CyclicLR is disabled when using stochastic weight averaging and snapshot + ensembling. + + Args: + cs (ConfigurationSpace): + Configuration space to which forbidden conditions are added. + + """ + + # Learned Entity Embedding is only valid when encoder is one hot encoder + if 'network_embedding' in self.named_steps.keys() and 'encoder' in self.named_steps.keys(): + embeddings = cs.get_hyperparameter('network_embedding:__choice__').choices + if 'LearnedEntityEmbedding' in embeddings: + encoders = cs.get_hyperparameter('encoder:__choice__').choices + possible_default_embeddings = copy(list(embeddings)) + del possible_default_embeddings[possible_default_embeddings.index('LearnedEntityEmbedding')] + + for encoder in encoders: + if encoder == 'OneHotEncoder': + continue + while True: + try: + cs.add_forbidden_clause(ForbiddenAndConjunction( + ForbiddenEqualsClause(cs.get_hyperparameter( + 'network_embedding:__choice__'), 'LearnedEntityEmbedding'), + ForbiddenEqualsClause(cs.get_hyperparameter('encoder:__choice__'), encoder) + )) + break + except ValueError: + # change the default and try again + try: + default = possible_default_embeddings.pop() + except IndexError: + raise ValueError("Cannot find a legal default configuration") + cs.get_hyperparameter('network_embedding:__choice__').default_value = default + + # Disable CyclicLR until todo is completed. + if 'lr_scheduler' in self.named_steps.keys() and 'trainer' in self.named_steps.keys(): + trainers = cs.get_hyperparameter('trainer:__choice__').choices + for trainer in trainers: + available_schedulers = cs.get_hyperparameter('lr_scheduler:__choice__').choices + # TODO: update cyclic lr to use n_restarts and adjust according to batch size + cyclic_lr_name = 'CyclicLR' + if cyclic_lr_name in available_schedulers: + # disable snapshot ensembles and stochastic weight averaging + cs.add_forbidden_clause(ForbiddenAndConjunction( + ForbiddenEqualsClause(cs.get_hyperparameter( + f'trainer:{trainer}:use_snapshot_ensemble'), True), + ForbiddenEqualsClause(cs.get_hyperparameter('lr_scheduler:__choice__'), cyclic_lr_name) + )) + cs.add_forbidden_clause(ForbiddenAndConjunction( + ForbiddenEqualsClause(cs.get_hyperparameter( + f'trainer:{trainer}:use_stochastic_weight_averaging'), True), + ForbiddenEqualsClause(cs.get_hyperparameter('lr_scheduler:__choice__'), cyclic_lr_name) + )) + return cs + def __repr__(self) -> str: """Retrieves a str representation of the current pipeline diff --git a/autoPyTorch/pipeline/components/setup/network_embedding/__init__.py b/autoPyTorch/pipeline/components/setup/network_embedding/__init__.py index 86b8b899d..0e79eedbc 100644 --- a/autoPyTorch/pipeline/components/setup/network_embedding/__init__.py +++ b/autoPyTorch/pipeline/components/setup/network_embedding/__init__.py @@ -146,71 +146,64 @@ def get_hyperparameter_search_space( if default is None: defaults = [ 'NoEmbedding', - # 'LearnedEntityEmbedding', + 'LearnedEntityEmbedding', ] for default_ in defaults: if default_ in available_embedding: default = default_ break - # Restrict embedding to NoEmbedding until preprocessing is fixed - embedding = CSH.CategoricalHyperparameter('__choice__', - ['NoEmbedding'], - default_value=default) + if isinstance(dataset_properties['categorical_columns'], list): + categorical_columns = dataset_properties['categorical_columns'] + else: + categorical_columns = [] + + updates = self._get_search_space_updates() + if '__choice__' in updates.keys(): + choice_hyperparameter = updates['__choice__'] + if not set(choice_hyperparameter.value_range).issubset(available_embedding): + raise ValueError("Expected given update for {} to have " + "choices in {} got {}".format(self.__class__.__name__, + available_embedding, + choice_hyperparameter.value_range)) + if len(categorical_columns) == 0: + assert len(choice_hyperparameter.value_range) == 1 + if 'NoEmbedding' not in choice_hyperparameter.value_range: + raise ValueError("Provided {} in choices, however, the dataset " + "is incompatible with it".format(choice_hyperparameter.value_range)) + embedding = CSH.CategoricalHyperparameter('__choice__', + choice_hyperparameter.value_range, + default_value=choice_hyperparameter.default_value) + else: + + if len(categorical_columns) == 0: + default = 'NoEmbedding' + if include is not None and default not in include: + raise ValueError("Provided {} in include, however, the dataset " + "is incompatible with it".format(include)) + embedding = CSH.CategoricalHyperparameter('__choice__', + ['NoEmbedding'], + default_value=default) + else: + embedding = CSH.CategoricalHyperparameter('__choice__', + list(available_embedding.keys()), + default_value=default) + cs.add_hyperparameter(embedding) + for name in embedding.choices: + updates = self._get_search_space_updates(prefix=name) + config_space = available_embedding[name].get_hyperparameter_search_space(dataset_properties, # type: ignore + **updates) + parent_hyperparameter = {'parent': embedding, 'value': name} + cs.add_configuration_space( + name, + config_space, + parent_hyperparameter=parent_hyperparameter + ) + self.configuration_space_ = cs self.dataset_properties_ = dataset_properties return cs - # categorical_columns = dataset_properties['categorical_columns'] \ - # if isinstance(dataset_properties['categorical_columns'], List) else [] - - # updates = self._get_search_space_updates() - # if '__choice__' in updates.keys(): - # choice_hyperparameter = updates['__choice__'] - # if not set(choice_hyperparameter.value_range).issubset(available_embedding): - # raise ValueError("Expected given update for {} to have " - # "choices in {} got {}".format(self.__class__.__name__, - # available_embedding, - # choice_hyperparameter.value_range)) - # if len(categorical_columns) == 0: - # assert len(choice_hyperparameter.value_range) == 1 - # if 'NoEmbedding' not in choice_hyperparameter.value_range: - # raise ValueError("Provided {} in choices, however, the dataset " - # "is incompatible with it".format(choice_hyperparameter.value_range)) - # embedding = CSH.CategoricalHyperparameter('__choice__', - # choice_hyperparameter.value_range, - # default_value=choice_hyperparameter.default_value) - # else: - - # if len(categorical_columns) == 0: - # default = 'NoEmbedding' - # if include is not None and default not in include: - # raise ValueError("Provided {} in include, however, the dataset " - # "is incompatible with it".format(include)) - # embedding = CSH.CategoricalHyperparameter('__choice__', - # ['NoEmbedding'], - # default_value=default) - # else: - # embedding = CSH.CategoricalHyperparameter('__choice__', - # list(available_embedding.keys()), - # default_value=default) - - # cs.add_hyperparameter(embedding) - # for name in embedding.choices: - # updates = self._get_search_space_updates(prefix=name) - # config_space = available_embedding[name].get_hyperparameter_search_space( - # dataset_properties, # type: ignore - # **updates) - # parent_hyperparameter = {'parent': embedding, 'value': name} - # cs.add_configuration_space( - # name, - # config_space, - # parent_hyperparameter=parent_hyperparameter - # ) - - # self.configuration_space_ = cs - # self.dataset_properties_ = dataset_properties - # return cs def transform(self, X: Dict[str, Any]) -> Dict[str, Any]: assert self.choice is not None, "Cannot call transform before the object is initialized" diff --git a/autoPyTorch/pipeline/components/training/trainer/base_trainer.py b/autoPyTorch/pipeline/components/training/trainer/base_trainer.py index a6602a88f..40f10317f 100644 --- a/autoPyTorch/pipeline/components/training/trainer/base_trainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/base_trainer.py @@ -342,6 +342,35 @@ def on_epoch_start(self, X: Dict[str, Any], epoch: int) -> None: """ pass + def _swa_update(self) -> None: + """ + perform swa model update + """ + if self.swa_model is None: + raise ValueError("SWA model cannot be none when stochastic weight averaging is enabled") + self.swa_model.update_parameters(self.model) + self.swa_updated = True + + def _se_update(self, epoch: int) -> None: + """ + Add latest model or swa_model to model snapshot ensemble + Args: + epoch (int): + current epoch + """ + if self.model_snapshots is None: + raise ValueError("model snapshots cannot be None when snapshot ensembling is enabled") + is_last_epoch = (epoch == self.budget_tracker.max_epochs) + if is_last_epoch and self.use_stochastic_weight_averaging: + model_copy = deepcopy(self.swa_model) + else: + model_copy = deepcopy(self.model) + + assert model_copy is not None + model_copy.cpu() + self.model_snapshots.append(model_copy) + self.model_snapshots = self.model_snapshots[-self.se_lastk:] + def on_epoch_end(self, X: Dict[str, Any], epoch: int) -> bool: """ Optional place holder for AutoPytorch Extensions. @@ -352,39 +381,19 @@ def on_epoch_end(self, X: Dict[str, Any], epoch: int) -> bool: if X['is_cyclic_scheduler']: if hasattr(self.scheduler, 'T_cur') and self.scheduler.T_cur == 0 and epoch != 1: if self.use_stochastic_weight_averaging: - assert self.swa_model is not None, "SWA model can't be none when" \ - " stochastic weight averaging is enabled" - self.swa_model.update_parameters(self.model) - self.swa_updated = True + self._swa_update() if self.use_snapshot_ensemble: - assert self.model_snapshots is not None, "model snapshots container can't be " \ - "none when snapshot ensembling is enabled" - is_last_epoch = (epoch == self.budget_tracker.max_epochs) - if is_last_epoch and self.use_stochastic_weight_averaging: - model_copy = deepcopy(self.swa_model) - else: - model_copy = deepcopy(self.model) - - assert model_copy is not None - model_copy.cpu() - self.model_snapshots.append(model_copy) - self.model_snapshots = self.model_snapshots[-self.se_lastk:] + self._se_update(epoch=epoch) else: - if epoch > self._budget_threshold: - if self.use_stochastic_weight_averaging: - assert self.swa_model is not None, "SWA model can't be none when" \ - " stochastic weight averaging is enabled" - self.swa_model.update_parameters(self.model) - self.swa_updated = True - if self.use_snapshot_ensemble: - assert self.model_snapshots is not None, "model snapshots container can't be " \ - "none when snapshot ensembling is enabled" - model_copy = deepcopy(self.swa_model) if self.use_stochastic_weight_averaging \ - else deepcopy(self.model) - assert model_copy is not None - model_copy.cpu() - self.model_snapshots.append(model_copy) - self.model_snapshots = self.model_snapshots[-self.se_lastk:] + if epoch > self._budget_threshold and self.use_stochastic_weight_averaging: + self._swa_update() + + if ( + self.use_snapshot_ensemble + and self.budget_tracker.max_epochs is not None + and epoch > (self.budget_tracker.max_epochs - self.se_lastk) + ): + self._se_update(epoch=epoch) return False def _scheduler_step( diff --git a/autoPyTorch/pipeline/image_classification.py b/autoPyTorch/pipeline/image_classification.py index 276e05816..13f8a4cf8 100644 --- a/autoPyTorch/pipeline/image_classification.py +++ b/autoPyTorch/pipeline/image_classification.py @@ -156,6 +156,7 @@ def _get_hyperparameter_search_space(self, # Here we add custom code, like this with this # is not a valid configuration + cs = self._add_forbidden_conditions(cs) self.configuration_space = cs self.dataset_properties = dataset_properties diff --git a/autoPyTorch/pipeline/tabular_classification.py b/autoPyTorch/pipeline/tabular_classification.py index 720d0af64..2e64a6944 100644 --- a/autoPyTorch/pipeline/tabular_classification.py +++ b/autoPyTorch/pipeline/tabular_classification.py @@ -3,7 +3,6 @@ from typing import Any, Dict, List, Optional, Tuple, Union from ConfigSpace.configuration_space import Configuration, ConfigurationSpace -from ConfigSpace.forbidden import ForbiddenAndConjunction, ForbiddenEqualsClause import numpy as np @@ -261,33 +260,9 @@ def _get_hyperparameter_search_space(self, cs=cs, dataset_properties=dataset_properties, exclude=exclude, include=include, pipeline=self.steps) - # Here we add custom code, that is used to ensure valid configurations, For example - # Learned Entity Embedding is only valid when encoder is one hot encoder - if 'network_embedding' in self.named_steps.keys() and 'encoder' in self.named_steps.keys(): - embeddings = cs.get_hyperparameter('network_embedding:__choice__').choices - if 'LearnedEntityEmbedding' in embeddings: - encoders = cs.get_hyperparameter('encoder:__choice__').choices - possible_default_embeddings = copy.copy(list(embeddings)) - del possible_default_embeddings[possible_default_embeddings.index('LearnedEntityEmbedding')] - - for encoder in encoders: - if encoder == 'OneHotEncoder': - continue - while True: - try: - cs.add_forbidden_clause(ForbiddenAndConjunction( - ForbiddenEqualsClause(cs.get_hyperparameter( - 'network_embedding:__choice__'), 'LearnedEntityEmbedding'), - ForbiddenEqualsClause(cs.get_hyperparameter('encoder:__choice__'), encoder) - )) - break - except ValueError: - # change the default and try again - try: - default = possible_default_embeddings.pop() - except IndexError: - raise ValueError("Cannot find a legal default configuration") - cs.get_hyperparameter('network_embedding:__choice__').default_value = default + # Here we add custom code, like this with this + # is not a valid configuration + cs = self._add_forbidden_conditions(cs) self.configuration_space = cs self.dataset_properties = dataset_properties diff --git a/autoPyTorch/pipeline/tabular_regression.py b/autoPyTorch/pipeline/tabular_regression.py index 06da9cabb..4737bf57d 100644 --- a/autoPyTorch/pipeline/tabular_regression.py +++ b/autoPyTorch/pipeline/tabular_regression.py @@ -3,7 +3,6 @@ from typing import Any, Dict, List, Optional, Tuple, Union from ConfigSpace.configuration_space import Configuration, ConfigurationSpace -from ConfigSpace.forbidden import ForbiddenAndConjunction, ForbiddenEqualsClause import numpy as np @@ -210,33 +209,7 @@ def _get_hyperparameter_search_space(self, # Here we add custom code, like this with this # is not a valid configuration - # Learned Entity Embedding is only valid when encoder is one hot encoder - if 'network_embedding' in self.named_steps.keys() and 'encoder' in self.named_steps.keys(): - embeddings = cs.get_hyperparameter('network_embedding:__choice__').choices - if 'LearnedEntityEmbedding' in embeddings: - encoders = cs.get_hyperparameter('encoder:__choice__').choices - default = cs.get_hyperparameter('network_embedding:__choice__').default_value - possible_default_embeddings = copy.copy(list(embeddings)) - del possible_default_embeddings[possible_default_embeddings.index(default)] - - for encoder in encoders: - if encoder == 'OneHotEncoder': - continue - while True: - try: - cs.add_forbidden_clause(ForbiddenAndConjunction( - ForbiddenEqualsClause(cs.get_hyperparameter( - 'network_embedding:__choice__'), 'LearnedEntityEmbedding'), - ForbiddenEqualsClause(cs.get_hyperparameter('encoder:__choice__'), encoder) - )) - break - except ValueError: - # change the default and try again - try: - default = possible_default_embeddings.pop() - except IndexError: - raise ValueError("Cannot find a legal default configuration") - cs.get_hyperparameter('network_embedding:__choice__').default_value = default + cs = self._add_forbidden_conditions(cs) self.configuration_space = cs self.dataset_properties = dataset_properties From 637a68be0933b084ead21c2330ce470705809c94 Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Wed, 9 Mar 2022 18:31:38 +0100 Subject: [PATCH 44/50] fixes after rebase --- autoPyTorch/pipeline/components/training/trainer/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autoPyTorch/pipeline/components/training/trainer/__init__.py b/autoPyTorch/pipeline/components/training/trainer/__init__.py index e6a7acfb4..b70467837 100755 --- a/autoPyTorch/pipeline/components/training/trainer/__init__.py +++ b/autoPyTorch/pipeline/components/training/trainer/__init__.py @@ -323,7 +323,7 @@ def prepare_trainer(self, X: Dict) -> None: scheduler=X['lr_scheduler'], task_type=STRING_TO_TASK_TYPES[X['dataset_properties']['task_type']], labels=labels, - step_interval=X['step_interval'] + step_interval=X['step_interval'], numerical_columns=X['dataset_properties']['numerical_columns'] if 'numerical_columns' in X[ 'dataset_properties'] else None ) From e69ff3bde45f8ba6547c61ae86927daed2846022 Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Tue, 26 Jul 2022 15:23:56 +0200 Subject: [PATCH 45/50] fix tests after rebase --- autoPyTorch/api/base_task.py | 2 + autoPyTorch/data/base_feature_validator.py | 1 + autoPyTorch/data/tabular_feature_validator.py | 28 ++++------ .../data/time_series_feature_validator.py | 4 +- autoPyTorch/datasets/resampling_strategy.py | 3 - autoPyTorch/evaluation/abstract_evaluator.py | 2 +- autoPyTorch/optimizer/smbo.py | 3 +- autoPyTorch/pipeline/base_pipeline.py | 7 --- .../feature_preprocessing/utils.py | 11 +++- .../setup/network/forecasting_architecture.py | 47 ++++++++++++++++ .../setup/network_backbone/utils.py | 5 -- .../base_network_embedding.py | 56 ++++++++----------- .../time_series_forecasting_data_loader.py | 16 +++++- .../ForecastingMixUpTrainer.py | 3 + .../ForecastingStandardTrainer.py | 3 + autoPyTorch/utils/common.py | 6 +- requirements.txt | 1 - setup.py | 2 +- test/test_api/test_api.py | 2 + test/test_data/test_feature_validator.py | 35 +++++++----- .../components/training/test_training.py | 2 +- .../test_tabular_classification.py | 4 +- 22 files changed, 145 insertions(+), 98 deletions(-) diff --git a/autoPyTorch/api/base_task.py b/autoPyTorch/api/base_task.py index 2fcf66bae..7fd17249e 100644 --- a/autoPyTorch/api/base_task.py +++ b/autoPyTorch/api/base_task.py @@ -1908,6 +1908,7 @@ def _init_ensemble_builder( # builder in the provide dask client required_dataset_properties = {'task_type': self.task_type, 'output_type': self.dataset.output_type} + proc_ensemble = EnsembleBuilderManager( start_time=time.time(), time_left_for_ensembles=time_left_for_ensembles, @@ -1928,6 +1929,7 @@ def _init_ensemble_builder( random_state=self.seed, precision=precision, logger_port=self._logger_port, + metrics_kwargs=self._metrics_kwargs ) self._stopwatch.stop_task(ensemble_task_name) diff --git a/autoPyTorch/data/base_feature_validator.py b/autoPyTorch/data/base_feature_validator.py index 0f2ba35ae..c4c414e18 100644 --- a/autoPyTorch/data/base_feature_validator.py +++ b/autoPyTorch/data/base_feature_validator.py @@ -49,6 +49,7 @@ def __init__( self.categories: List[List[int]] = [] self.categorical_columns: List[int] = [] self.numerical_columns: List[int] = [] + self.encode_columns: List[int] = [] self.all_nan_columns: Optional[Set[Union[int, str]]] = None diff --git a/autoPyTorch/data/tabular_feature_validator.py b/autoPyTorch/data/tabular_feature_validator.py index e38a7eb22..5a7ce6f8b 100644 --- a/autoPyTorch/data/tabular_feature_validator.py +++ b/autoPyTorch/data/tabular_feature_validator.py @@ -168,27 +168,23 @@ def _fit( # Handle objects if possible exist_object_columns = has_object_columns(X.dtypes.values) + if exist_object_columns: X = self.infer_objects(X) self.dtypes = [dt.name for dt in X.dtypes] # Also note this change in self.dtypes + self.all_nan_columns = set(all_nan_columns) - self.transformed_columns, self.feat_types = self.get_columns_to_encode(X) + self.encode_columns, self.feat_types = self.get_columns_to_encode(X) assert self.feat_types is not None - preprocessors = get_tabular_preprocessors() - self.column_transformer = _create_column_transformer( - preprocessors=preprocessors, - categorical_columns=self.transformed_columns, - ) - - if len(self.enc_columns) > 0: + if len(self.encode_columns) > 0: preprocessors = get_tabular_preprocessors() self.column_transformer = _create_column_transformer( preprocessors=preprocessors, - categorical_columns=self.enc_columns, + categorical_columns=self.encode_columns, ) # Mypy redefinition @@ -302,8 +298,8 @@ def transform( # we change those columns to `object` dtype # to ensure that these columns are changed to appropriate dtype # in self.infer_objects - all_nan_cat_cols = set(X[self.enc_columns].columns[X[self.enc_columns].isna().all()]) - dtype_dict = {col: 'object' for col in self.enc_columns if col in all_nan_cat_cols} + all_nan_cat_cols = set(X[self.encode_columns].columns[X[self.encode_columns].isna().all()]) + dtype_dict = {col: 'object' for col in self.encode_columns if col in all_nan_cat_cols} X = X.astype(dtype_dict) # Check the data here so we catch problems on new test data @@ -388,10 +384,6 @@ def _check_data( if exist_object_columns: X = self.infer_objects(X) - # Define the column to be encoded here as the feature validator is fitted once - # per estimator - self.transformed_columns, self.feat_types = self.get_columns_to_encode(X) - column_order = [column for column in X.columns] if len(self.column_order) > 0: if self.column_order != column_order: @@ -491,8 +483,8 @@ def _get_columns_to_encode( Type of each column numerical/categorical """ - if len(self.transformed_columns) > 0 and self.feat_types is not None: - return self.transformed_columns, self.feat_types + if len(self.encode_columns) > 0 and self.feat_types is not None: + return self.encode_columns, self.feat_types # Register if a column needs encoding categorical_columns = [] @@ -503,7 +495,7 @@ def _get_columns_to_encode( for i, column in enumerate(X.columns): if self.all_nan_columns is not None and column in self.all_nan_columns: continue - column_dtype = self.dtypes[i] + column_dtype = self.dtypes[i] if len(self.dtypes) > 0 else X[column].dtype.name err_msg = "Valid types are `numerical`, `categorical` or `boolean`, " \ "but input column {} has an invalid type `{}`.".format(column, column_dtype) if column_dtype in ['category', 'bool']: diff --git a/autoPyTorch/data/time_series_feature_validator.py b/autoPyTorch/data/time_series_feature_validator.py index 962da78a8..f8b1c6724 100644 --- a/autoPyTorch/data/time_series_feature_validator.py +++ b/autoPyTorch/data/time_series_feature_validator.py @@ -37,8 +37,8 @@ def __init__( self.series_idx: Optional[List[Union[str, int]]] = None def get_reordered_columns(self) -> List[str]: - return self.transformed_columns + [ - col for col in self.column_order if col not in set(self.transformed_columns) + return self.encode_columns + [ + col for col in self.column_order if col not in set(self.encode_columns) ] def fit( diff --git a/autoPyTorch/datasets/resampling_strategy.py b/autoPyTorch/datasets/resampling_strategy.py index e2ac2736b..a85207087 100644 --- a/autoPyTorch/datasets/resampling_strategy.py +++ b/autoPyTorch/datasets/resampling_strategy.py @@ -109,10 +109,7 @@ def is_stratified(self) -> bool: # TODO: replace it with another way ResamplingStrategies = Union[CrossValTypes, HoldoutValTypes, NoResamplingStrategyTypes] -<<<<<<< HEAD -======= ->>>>>>> Additional metrics during train (#194) DEFAULT_RESAMPLING_PARAMETERS: Dict[ ResamplingStrategies, diff --git a/autoPyTorch/evaluation/abstract_evaluator.py b/autoPyTorch/evaluation/abstract_evaluator.py index c45e4db3c..c657f7784 100644 --- a/autoPyTorch/evaluation/abstract_evaluator.py +++ b/autoPyTorch/evaluation/abstract_evaluator.py @@ -729,7 +729,7 @@ def _loss(self, y_true: np.ndarray, y_hat: np.ndarray, **metric_kwargs: Any) -> def finish_up(self, loss: Dict[str, float], train_loss: Dict[str, float], valid_pred: Optional[np.ndarray], test_pred: Optional[np.ndarray], additional_run_info: Optional[Dict], file_output: bool, status: StatusType, - opt_pred: Optional[np.ndarray], + opt_pred: Optional[np.ndarray], **metric_kwargs: Any ) -> Optional[Tuple[float, float, int, Dict]]: """This function does everything necessary after the fitting is done: diff --git a/autoPyTorch/optimizer/smbo.py b/autoPyTorch/optimizer/smbo.py index fefa5cc12..e88449ed6 100644 --- a/autoPyTorch/optimizer/smbo.py +++ b/autoPyTorch/optimizer/smbo.py @@ -276,7 +276,8 @@ def __init__(self, initial_configurations = [] if STRING_TO_TASK_TYPES.get(self.task_type, -1) == TIMESERIES_FORECASTING: - initial_configurations = self.get_init_configs_for_forecasting(config_space, kwargs) + # TODO: update search space (to remove reg cocktails) for forecasting tasks so that we can use the portfolio (or build the portfolio again) + # initial_configurations = self.get_init_configs_for_forecasting(config_space, kwargs) # proxy-validation sets self.min_num_test_instances: Optional[int] = kwargs.get('min_num_test_instances', # type:ignore[assignment] None) diff --git a/autoPyTorch/pipeline/base_pipeline.py b/autoPyTorch/pipeline/base_pipeline.py index e48f37510..7bc3bd454 100644 --- a/autoPyTorch/pipeline/base_pipeline.py +++ b/autoPyTorch/pipeline/base_pipeline.py @@ -520,7 +520,6 @@ def _check_search_space_updates(self, include: Optional[Dict[str, Any]], # needs to be updated is in components of the # choice module elif split_hyperparameter[0] not in components.keys(): -<<<<<<< HEAD hp_in_component = False if hasattr(node, 'additional_components') and node.additional_components: # This is designed for forecasting network encoder: @@ -538,12 +537,6 @@ def _check_search_space_updates(self, include: Optional[Dict[str, Any]], "Expected update hyperparameter " "to be in {} got {}".format(node.__class__.__name__, components.keys(), split_hyperparameter[0])) -======= - raise ValueError("Unknown component choice for node {}. " - "Expected update component " - "to be in {}, but got {}".format(node_name, - components.keys(), split_hyperparameter[0])) ->>>>>>> Bug fixes (#249) else: # check if hyperparameter is in the search space of the component component = components[split_hyperparameter[0]] diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/utils.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/utils.py index 5d91ac2b6..a8c57959e 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/utils.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/utils.py @@ -81,11 +81,18 @@ def percentage_value_range_to_integer_range( log = False else: log = hyperparameter_search_space.log + + min_hyperparameter_value = hyperparameter_search_space.value_range[0] + if len(hyperparameter_search_space.value_range) > 1: + max_hyperparameter_value = hyperparameter_search_space.value_range[1] + else: + max_hyperparameter_value = hyperparameter_search_space.value_range[0] + hyperparameter_search_space = HyperparameterSearchSpace( hyperparameter=hyperparameter_name, value_range=( - floor(float(hyperparameter_search_space.value_range[0]) * n_features), - floor(float(hyperparameter_search_space.value_range[1]) * n_features)), + floor(float(min_hyperparameter_value) * n_features), + floor(float(max_hyperparameter_value) * n_features)), default_value=ceil(float(hyperparameter_search_space.default_value) * n_features), log=log) else: diff --git a/autoPyTorch/pipeline/components/setup/network/forecasting_architecture.py b/autoPyTorch/pipeline/components/setup/network/forecasting_architecture.py index fc7ac3ae1..57026728c 100644 --- a/autoPyTorch/pipeline/components/setup/network/forecasting_architecture.py +++ b/autoPyTorch/pipeline/components/setup/network/forecasting_architecture.py @@ -574,6 +574,10 @@ def forward(self, past_observed_targets: Optional[torch.BoolTensor] = None, decoder_observed_values: Optional[torch.Tensor] = None, ) -> ALL_NET_OUTPUT: + + if isinstance(past_targets, dict): + past_targets, past_features, future_features, past_observed_targets = self._unwrap_past_targets(past_targets) + x_past, x_future, x_static, loc, scale, static_context_initial_hidden, _ = self.pre_processing( past_targets=past_targets, past_observed_targets=past_observed_targets, @@ -603,6 +607,38 @@ def forward(self, return self.rescale_output(output, loc, scale, self.device) + def _unwrap_past_targets( + self, + past_targets: dict + ) -> Tuple[ + torch.Tensor, + Optional[torch.Tensor], + Optional[torch.Tensor], + Optional[torch.Tensor], + Optional[torch.BoolTensor], + Optional[torch.Tensor]]: + """ + Time series forecasting network requires multiple inputs for the forward pass which is different to how pytorch + networks usually work. SWA's update_bn in line #452 of trainer choice, does not unwrap the dictionary of the + input when running the forward pass. So we need to check for that here. + + Args: + past_targets (dict): + Input mistakenly passed to past_targets variable + + Returns: + _type_: _description_ + """ + + past_targets_copy = past_targets.copy() + past_targets = past_targets_copy.pop('past_targets') + future_targets = past_targets_copy.pop('future_targets', None) + past_features = past_targets_copy.pop('past_features', None) + future_features = past_targets_copy.pop('future_features', None) + past_observed_targets = past_targets_copy.pop('past_observed_targets', None) + decoder_observed_values = past_targets_copy.pop('decoder_observed_values', None) + return past_targets,past_features,future_features,past_observed_targets + def pred_from_net_output(self, net_output: ALL_NET_OUTPUT) -> torch.Tensor: if self.output_type == 'regression': return net_output @@ -694,6 +730,10 @@ def forward(self, future_features: Optional[torch.Tensor] = None, past_observed_targets: Optional[torch.BoolTensor] = None, decoder_observed_values: Optional[torch.Tensor] = None, ) -> ALL_NET_OUTPUT: + + if isinstance(past_targets, dict): + past_targets, past_features, future_features, past_observed_targets = self._unwrap_past_targets(past_targets) + x_past, _, x_static, loc, scale, static_context_initial_hidden, past_targets = self.pre_processing( past_targets=past_targets, past_observed_targets=past_observed_targets, @@ -983,6 +1023,10 @@ def forward(self, future_features: Optional[torch.Tensor] = None, past_observed_targets: Optional[torch.BoolTensor] = None, decoder_observed_values: Optional[torch.Tensor] = None, ) -> ALL_NET_OUTPUT: + + if isinstance(past_targets, dict): + past_targets, past_features, future_features, past_observed_targets = self._unwrap_past_targets(past_targets) + encode_length = min(self.window_size, past_targets.shape[1]) if past_observed_targets is None: @@ -1250,6 +1294,9 @@ def forward(self, # type: ignore[override] decoder_observed_values: Optional[torch.Tensor] = None, ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + if isinstance(past_targets, dict): + past_targets, past_features, future_features, past_observed_targets = self._unwrap_past_targets(past_targets) + # Unlike other networks, NBEATS network is required to predict both past and future targets. # Thereby, we return two tensors for backcast and forecast if past_observed_targets is None: diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/utils.py b/autoPyTorch/pipeline/components/setup/network_backbone/utils.py index 05e39fd09..a3216c7c1 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/utils.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/utils.py @@ -29,15 +29,10 @@ def get_output_shape(network: torch.nn.Module, input_shape: Tuple[int, ...], has """ placeholder = torch.randn((2, *input_shape), dtype=torch.float) with torch.no_grad(): -<<<<<<< HEAD if has_hidden_states: output = network(placeholder)[0] else: output = network(placeholder) -======= - output = network(placeholder) - ->>>>>>> Bug fixes (#249) return tuple(output.shape[1:]) diff --git a/autoPyTorch/pipeline/components/setup/network_embedding/base_network_embedding.py b/autoPyTorch/pipeline/components/setup/network_embedding/base_network_embedding.py index b99f253f3..1ff5df13e 100644 --- a/autoPyTorch/pipeline/components/setup/network_embedding/base_network_embedding.py +++ b/autoPyTorch/pipeline/components/setup/network_embedding/base_network_embedding.py @@ -11,14 +11,15 @@ class NetworkEmbeddingComponent(autoPyTorchSetupComponent): - def __init__(self, random_state: Optional[np.random.RandomState] = None): - super().__init__(random_state=random_state) + def __init__(self, random_state: Optional[Union[np.random.RandomState, int]] = None): + super().__init__() self.embedding: Optional[nn.Module] = None + self.random_state = random_state self.feature_shapes: Dict[str, int] = {} def fit(self, X: Dict[str, Any], y: Any = None) -> BaseEstimator: - num_numerical_columns, num_input_features = self._get_required_info_from_data(X) + num_numerical_columns, num_input_features = self._get_args(X) self.embedding, num_output_features = self.build_embedding( num_input_features=num_input_features, @@ -35,7 +36,6 @@ def fit(self, X: Dict[str, Any], y: Any = None) -> BaseEstimator: self.feature_shapes = feature_shapes else: self.feature_shapes = X['dataset_properties']['feature_shapes'] - return self def transform(self, X: Dict[str, Any]) -> Dict[str, Any]: @@ -49,39 +49,31 @@ def build_embedding(self, num_numerical_features: int) -> Tuple[nn.Module, Optional[List[int]]]: raise NotImplementedError - def _get_required_info_from_data(self, X: Dict[str, Any]) -> Tuple[int, np.ndarray]: - """ - Returns the number of numerical columns after preprocessing and - an array of size equal to the number of input features - containing zeros for numerical data and number of categories - for categorical data. This is required to build the embedding. - - Args: - X (Dict[str, Any]): - Fit dictionary - - Returns: - Tuple[int, np.ndarray]: - number of numerical columns and array indicating - number of categories for categorical columns and - 0 for numerical columns - """ + def _get_args(self, X: Dict[str, Any]) -> Tuple[int, np.ndarray]: # Feature preprocessors can alter numerical columns if len(X['dataset_properties']['numerical_columns']) == 0: num_numerical_columns = 0 else: X_train = copy.deepcopy(X['backend'].load_datamanager().train_tensors[0][:2]) - numerical_column_transformer = X['tabular_transformer'].preprocessor. \ - named_transformers_['numerical_pipeline'] - num_numerical_columns = numerical_column_transformer.transform( - X_train[:, X['dataset_properties']['numerical_columns']]).shape[1] - - num_cols = num_numerical_columns + len(X['dataset_properties']['categorical_columns']) - num_input_feats = np.zeros(num_cols, dtype=np.int32) - + if 'tabular_transformer' in X: + numerical_column_transformer = X['tabular_transformer'].preprocessor. \ + named_transformers_['numerical_pipeline'] + elif 'time_series_feature_transformer' in X: + numerical_column_transformer = X['time_series_feature_transformer'].preprocessor. \ + named_transformers_['numerical_pipeline'] + else: + raise ValueError("Either a tabular or time_series transformer must be contained!") + if hasattr(X_train, 'iloc'): + num_numerical_columns = numerical_column_transformer.transform( + X_train.iloc[:, X['dataset_properties']['numerical_columns']]).shape[1] + else: + num_numerical_columns = numerical_column_transformer.transform( + X_train[:, X['dataset_properties']['numerical_columns']]).shape[1] + num_input_features = np.zeros((num_numerical_columns + len(X['dataset_properties']['categorical_columns'])), + dtype=np.int32) categories = X['dataset_properties']['categories'] - for idx, cats in enumerate(categories, start=num_numerical_columns): - num_input_feats[idx] = len(cats) - return num_numerical_columns, num_input_feats + for i, category in enumerate(categories): + num_input_features[num_numerical_columns + i, ] = len(category) + return num_numerical_columns, num_input_features diff --git a/autoPyTorch/pipeline/components/training/data_loader/time_series_forecasting_data_loader.py b/autoPyTorch/pipeline/components/training/data_loader/time_series_forecasting_data_loader.py index 3ddd66b2a..92c16c1d5 100644 --- a/autoPyTorch/pipeline/components/training/data_loader/time_series_forecasting_data_loader.py +++ b/autoPyTorch/pipeline/components/training/data_loader/time_series_forecasting_data_loader.py @@ -254,8 +254,7 @@ def fit(self, X: Dict[str, Any], y: Any = None) -> torch.utils.data.DataLoader: self.val_transform, train=False, ) - - if X['dataset_properties']["is_small_preprocess"]: + if X['dataset_properties'].get("is_small_preprocess", True): # This parameter indicates that the data has been pre-processed for speed # Overwrite the datamanager with the pre-processes data datamanager.replace_data(X['X_train'], @@ -616,3 +615,16 @@ def __str__(self) -> str: """ Allow a nice understanding of what components where used """ string = self.train_data_loader.__class__.__name__ return string + + def _check_transform_requirements(self, X: Dict[str, Any], y: Any = None) -> None: + """ + + Makes sure that the fit dictionary contains the required transformations + that the dataset should go through + + Args: + X (Dict[str, Any]): Dictionary with fitted parameters. It is a message passing + mechanism, in which during a transform, a components adds relevant information + so that further stages can be properly fitted + """ + pass diff --git a/autoPyTorch/pipeline/components/training/trainer/forecasting_trainer/ForecastingMixUpTrainer.py b/autoPyTorch/pipeline/components/training/trainer/forecasting_trainer/ForecastingMixUpTrainer.py index 197887339..47510857a 100644 --- a/autoPyTorch/pipeline/components/training/trainer/forecasting_trainer/ForecastingMixUpTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/forecasting_trainer/ForecastingMixUpTrainer.py @@ -13,4 +13,7 @@ def get_properties(dataset_properties: Optional[Dict[str, BaseDatasetPropertiesT return { 'shortname': 'ForecastingMixUpTrainer', 'name': 'MixUp Regularized Trainer', + 'handles_tabular': False, + 'handles_image': False, + 'handles_time_series': True, } diff --git a/autoPyTorch/pipeline/components/training/trainer/forecasting_trainer/ForecastingStandardTrainer.py b/autoPyTorch/pipeline/components/training/trainer/forecasting_trainer/ForecastingStandardTrainer.py index 9235565fe..6b92c9513 100644 --- a/autoPyTorch/pipeline/components/training/trainer/forecasting_trainer/ForecastingStandardTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/forecasting_trainer/ForecastingStandardTrainer.py @@ -13,4 +13,7 @@ def get_properties(dataset_properties: Optional[Dict[str, BaseDatasetPropertiesT return { 'shortname': 'ForecastingStandardTrainer', 'name': 'Forecasting Standard Trainer', + 'handles_tabular': False, + 'handles_image': False, + 'handles_time_series': True, } diff --git a/autoPyTorch/utils/common.py b/autoPyTorch/utils/common.py index 552612c97..a13bec3fe 100644 --- a/autoPyTorch/utils/common.py +++ b/autoPyTorch/utils/common.py @@ -105,9 +105,6 @@ def __str__(self) -> str: return str(self.value) -<<<<<<< HEAD -def custom_collate_fn(batch: List, x_collector: Callable = default_collate) -> List[Optional[torch.Tensor]]: -======= def replace_prefix_in_config_dict(config: Dict[str, Any], prefix: str, replace: str = "") -> Dict[str, Any]: """ Replace the prefix in all keys with the specified replacement string (the empty string by @@ -128,8 +125,7 @@ def replace_prefix_in_config_dict(config: Dict[str, Any], prefix: str, replace: k.startswith(prefix)} -def custom_collate_fn(batch: List) -> List[Optional[torch.Tensor]]: ->>>>>>> Bug fixes (#249) +def custom_collate_fn(batch: List, x_collector: Callable = default_collate) -> List[Optional[torch.Tensor]]: """ In the case of not providing a y tensor, in a dataset of form {X, y}, y would be None. diff --git a/requirements.txt b/requirements.txt index 2a76f011a..29690c18b 100755 --- a/requirements.txt +++ b/requirements.txt @@ -14,6 +14,5 @@ smac>=1.2 dask distributed>=2.2.0 catboost -lightgbm flaky tabulate \ No newline at end of file diff --git a/setup.py b/setup.py index bd6fa5b11..40e237349 100755 --- a/setup.py +++ b/setup.py @@ -64,7 +64,7 @@ "pytest-cov", 'pytest-forked', 'pytest-subtests', - "pytest-mock" + "pytest-mock", "codecov", "pep8", "mypy", diff --git a/test/test_api/test_api.py b/test/test_api/test_api.py index c043b2d57..12b12c3ad 100644 --- a/test/test_api/test_api.py +++ b/test/test_api/test_api.py @@ -460,6 +460,7 @@ def test_time_series_forecasting(forecasting_toy_dataset, resampling_strategy, b resampling_strategy_args=resampling_strategy_args, ensemble_size=2, seed=42, + delete_tmp_folder_after_terminate=False ) with unittest.mock.patch.object(estimator, '_do_dummy_prediction', new=dummy_do_dummy_prediction): @@ -475,6 +476,7 @@ def test_time_series_forecasting(forecasting_toy_dataset, resampling_strategy, b total_walltime_limit=30, func_eval_time_limit_secs=10, known_future_features=known_future_features, + enable_traditional_pipeline=False ) # Internal dataset has expected settings diff --git a/test/test_data/test_feature_validator.py b/test/test_data/test_feature_validator.py index 293d48de6..f3d511a79 100644 --- a/test/test_data/test_feature_validator.py +++ b/test/test_data/test_feature_validator.py @@ -139,9 +139,9 @@ def test_featurevalidator_fitontypeA_transformtypeB(input_data_featuretest): if isinstance(input_data_featuretest, pd.DataFrame): pytest.skip("Column order change in pandas is not supported") elif isinstance(input_data_featuretest, np.ndarray): - complementary_type = validator.numpy_array_to_pandas(input_data_featuretest) + complementary_type = validator.numpy_to_pandas(input_data_featuretest) elif isinstance(input_data_featuretest, list): - complementary_type, _ = validator.list_to_dataframe(input_data_featuretest) + complementary_type, _ = validator.list_to_pandas(input_data_featuretest) elif sparse.issparse(input_data_featuretest): complementary_type = sparse.csr_matrix(input_data_featuretest.todense()) else: @@ -167,9 +167,7 @@ def test_featurevalidator_get_columns_to_encode(): for col in df.columns: df[col] = df[col].astype(col) - validator.fit(df) - - categorical_columns, feat_type = validator._get_columns_info(df) + categorical_columns, feat_type = validator.get_columns_to_encode(df) assert categorical_columns == ['category', 'bool'] assert feat_type == ['numerical', 'numerical', 'categorical', 'categorical'] @@ -290,18 +288,25 @@ def test_features_unsupported_calls_are_raised(): expected """ validator = TabularFeatureValidator() - with pytest.raises(ValueError, match=r"AutoPyTorch does not support time"): + with pytest.raises(TypeError, match=r"Valid types are `numerical`, `categorical` or `boolean`, but input column"): validator.fit( pd.DataFrame({'datetime': [pd.Timestamp('20180310')]}) ) + + validator = TabularFeatureValidator() with pytest.raises(ValueError, match=r"AutoPyTorch only supports.*yet, the provided input"): validator.fit({'input1': 1, 'input2': 2}) - with pytest.raises(ValueError, match=r"has unsupported dtype string"): + + validator = TabularFeatureValidator() + with pytest.raises(TypeError, match=r"Valid types are `numerical`, `categorical` or `boolean`, but input column"): validator.fit(pd.DataFrame([{'A': 1, 'B': 2}], dtype='string')) + + validator = TabularFeatureValidator() with pytest.raises(ValueError, match=r"The feature dimensionality of the train and test"): validator.fit(X_train=np.array([[1, 2, 3], [4, 5, 6]]), X_test=np.array([[1, 2, 3, 4], [4, 5, 6, 7]]), ) + validator = TabularFeatureValidator() with pytest.raises(ValueError, match=r"Cannot call transform on a validator that is not fit"): validator.transform(np.array([[1, 2, 3], [4, 5, 6]])) @@ -366,15 +371,15 @@ def test_column_transformer_created(input_data_featuretest): # Make sure that the encoded features are actually encoded. Categorical columns are at # the start after transformation. In our fixtures, this is also honored prior encode - cat_columns, feature_types = validator._get_columns_info(input_data_featuretest) + cat_columns, feature_types = validator.get_columns_to_encode(input_data_featuretest) # At least one categorical - assert 'categorical' in validator.feat_typess + assert 'categorical' in validator.feat_types # Numerical if the original data has numerical only columns if np.any([pd.api.types.is_numeric_dtype(input_data_featuretest[col] ) for col in input_data_featuretest.columns]): - assert 'numerical' in validator.feat_typess + assert 'numerical' in validator.feat_types for i, feat_type in enumerate(feature_types): if 'numerical' in feat_type: np.testing.assert_array_equal( @@ -480,13 +485,13 @@ def test_feature_validator_new_data_after_fit( if train_data_type == 'pandas': old_dtypes = copy.deepcopy(validator.dtypes) validator.dtypes = ['dummy' for dtype in X_train.dtypes] - with pytest.raises(ValueError, match=r"Changing the dtype of the features after fit"): + with pytest.raises(ValueError, match=r"The dtype of the features must not be changed after fit()"): transformed_X = validator.transform(X_test) validator.dtypes = old_dtypes if test_data_type == 'pandas': columns = X_test.columns.tolist() X_test = X_test[reversed(columns)] - with pytest.raises(ValueError, match=r"Changing the column order of the features"): + with pytest.raises(ValueError, match=r"The column order of the features"): transformed_X = validator.transform(X_test) @@ -640,8 +645,7 @@ def test_feature_validator_get_columns_to_encode_error_feat_type(input_data_feat with pytest.raises(ValueError, match=r"Expected type of features to be in .*"): validator._validate_feat_types(X) -def test_feature_validator_imbalanced_data(): - + # Null columns in the train split but not necessarily in the test split train_features = { 'A': [np.NaN, np.NaN, np.NaN], @@ -662,7 +666,7 @@ def test_feature_validator_imbalanced_data(): validator.fit(X_train) train_feature_types = copy.deepcopy(validator.feat_types) - assert train_feature_types == ['numerical', 'numerical', 'numerical', 'numerical'] + assert train_feature_types == ['numerical'] # validator will throw an error if the column types are not the same transformed_X_test = validator.transform(X_test) transformed_X_test = pd.DataFrame(transformed_X_test) @@ -693,6 +697,7 @@ def test_feature_validator_imbalanced_data(): train_feature_types = copy.deepcopy(validator.feat_types) assert train_feature_types == ['categorical', 'numerical', 'numerical'] + null_columns = [] transformed_X_test = validator.transform(X_test) transformed_X_test = pd.DataFrame(transformed_X_test) assert not len(validator.all_nan_columns) diff --git a/test/test_pipeline/components/training/test_training.py b/test/test_pipeline/components/training/test_training.py index c011cea38..ae85cad4d 100644 --- a/test/test_pipeline/components/training/test_training.py +++ b/test/test_pipeline/components/training/test_training.py @@ -392,7 +392,7 @@ def test_every_trainer_is_valid(): @pytest.mark.parametrize("test_input,expected", [ ("tabular_classification", set(['RowCutMixTrainer', 'RowCutOutTrainer', 'AdversarialTrainer'])), ("image_classification", set(['GridCutMixTrainer', 'GridCutOutTrainer', 'AdversarialTrainer'])), - ("time_series_classification", set([])), + ("time_series_forecasting", set([])), ]) def test_get_set_config_space(test_input, expected): """Make sure that we can setup a valid choice in the trainer diff --git a/test/test_pipeline/test_tabular_classification.py b/test/test_pipeline/test_tabular_classification.py index e2b14c59f..20b6e1c46 100644 --- a/test/test_pipeline/test_tabular_classification.py +++ b/test/test_pipeline/test_tabular_classification.py @@ -593,8 +593,8 @@ def test_train_pipeline_with_runtime(fit_dictionary_tabular_dummy): # There is no epoch limitation assert not budget_tracker.is_max_epoch_reached(epoch=np.inf) - # More than 200 epochs would have pass in 5 seconds for this dataset - assert len(run_summary.performance_tracker['start_time']) > 100 + # More than 50 epochs would have pass in 5 seconds for this dataset + assert len(run_summary.performance_tracker['start_time']) > 50 @pytest.mark.parametrize("fit_dictionary_tabular_dummy", ["classification"], indirect=True) From c138173932582287e55f464c3e742907ef752b27 Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Tue, 26 Jul 2022 15:43:09 +0200 Subject: [PATCH 46/50] fix mypy and flake --- autoPyTorch/api/base_task.py | 2 +- autoPyTorch/api/time_series_forecasting.py | 6 ++ autoPyTorch/data/base_feature_validator.py | 2 +- autoPyTorch/data/tabular_feature_validator.py | 3 +- autoPyTorch/optimizer/smbo.py | 3 +- autoPyTorch/pipeline/base_pipeline.py | 8 ++- .../setup/network/forecasting_architecture.py | 60 +++++++++++++++---- .../pipeline/tabular_classification.py | 1 - autoPyTorch/pipeline/tabular_regression.py | 1 - test/test_data/test_feature_validator.py | 23 ------- .../test_tabular_classification.py | 5 +- 11 files changed, 68 insertions(+), 46 deletions(-) diff --git a/autoPyTorch/api/base_task.py b/autoPyTorch/api/base_task.py index 7fd17249e..303cefc4e 100644 --- a/autoPyTorch/api/base_task.py +++ b/autoPyTorch/api/base_task.py @@ -1908,7 +1908,7 @@ def _init_ensemble_builder( # builder in the provide dask client required_dataset_properties = {'task_type': self.task_type, 'output_type': self.dataset.output_type} - + proc_ensemble = EnsembleBuilderManager( start_time=time.time(), time_left_for_ensembles=time_left_for_ensembles, diff --git a/autoPyTorch/api/time_series_forecasting.py b/autoPyTorch/api/time_series_forecasting.py index 27b923576..67f6e5eaa 100644 --- a/autoPyTorch/api/time_series_forecasting.py +++ b/autoPyTorch/api/time_series_forecasting.py @@ -526,6 +526,9 @@ def predict( predicted value, it needs to be with shape (B, H, N), B is the number of series, H is forecasting horizon (n_prediction_steps), N is the number of targets """ + if self.dataset is None: + raise AttributeError(f"Expected dataset to be initialised when predicting in {self.__class__.__name__}") + if X_test is None or not isinstance(X_test[0], TimeSeriesSequence): assert past_targets is not None # Validate and construct TimeSeriesSequence @@ -566,6 +569,9 @@ def update_sliding_window_size(self, n_prediction_steps: int) -> None: forecast horizon. Sometimes we could also make our base sliding window size based on the forecast horizon """ + if self.dataset is None: + raise AttributeError(f"Expected dataset to be initialised when updating sliding window" + f" in {self.__class__.__name__}") base_window_size = int(np.ceil(self.dataset.base_window_size)) # we don't want base window size to large, which might cause a too long computation time, in which case # we will use n_prediction_step instead (which is normally smaller than base_window_size) diff --git a/autoPyTorch/data/base_feature_validator.py b/autoPyTorch/data/base_feature_validator.py index c4c414e18..8f65f8607 100644 --- a/autoPyTorch/data/base_feature_validator.py +++ b/autoPyTorch/data/base_feature_validator.py @@ -49,7 +49,7 @@ def __init__( self.categories: List[List[int]] = [] self.categorical_columns: List[int] = [] self.numerical_columns: List[int] = [] - self.encode_columns: List[int] = [] + self.encode_columns: List[str] = [] self.all_nan_columns: Optional[Set[Union[int, str]]] = None diff --git a/autoPyTorch/data/tabular_feature_validator.py b/autoPyTorch/data/tabular_feature_validator.py index 5a7ce6f8b..3beb19cba 100644 --- a/autoPyTorch/data/tabular_feature_validator.py +++ b/autoPyTorch/data/tabular_feature_validator.py @@ -283,7 +283,6 @@ def transform( X = self.numpy_to_pandas(X) if ispandas(X) and not issparse(X): - X = cast(pd.DataFrame, X) if self.all_nan_columns is None: raise ValueError('_fit must be called before calling transform') @@ -491,7 +490,7 @@ def _get_columns_to_encode( # Also, register the feature types for the estimator feat_types = [] - # Make sure each column is a valid type + # Make sure each column is a valid type for i, column in enumerate(X.columns): if self.all_nan_columns is not None and column in self.all_nan_columns: continue diff --git a/autoPyTorch/optimizer/smbo.py b/autoPyTorch/optimizer/smbo.py index e88449ed6..43b2c80c8 100644 --- a/autoPyTorch/optimizer/smbo.py +++ b/autoPyTorch/optimizer/smbo.py @@ -276,7 +276,8 @@ def __init__(self, initial_configurations = [] if STRING_TO_TASK_TYPES.get(self.task_type, -1) == TIMESERIES_FORECASTING: - # TODO: update search space (to remove reg cocktails) for forecasting tasks so that we can use the portfolio (or build the portfolio again) + # TODO: update search space (to remove reg cocktails) for forecasting tasks so + # that we can use the portfolio (or build the portfolio again) # initial_configurations = self.get_init_configs_for_forecasting(config_space, kwargs) # proxy-validation sets self.min_num_test_instances: Optional[int] = kwargs.get('min_num_test_instances', # type:ignore[assignment] diff --git a/autoPyTorch/pipeline/base_pipeline.py b/autoPyTorch/pipeline/base_pipeline.py index 7bc3bd454..6ded2adf6 100644 --- a/autoPyTorch/pipeline/base_pipeline.py +++ b/autoPyTorch/pipeline/base_pipeline.py @@ -1,7 +1,7 @@ -from copy import copy import warnings from abc import ABCMeta from collections import Counter +from copy import copy from typing import Any, Dict, List, Optional, Tuple, Union from ConfigSpace import Configuration @@ -297,7 +297,7 @@ def _get_hyperparameter_search_space(self, """ raise NotImplementedError() - def _add_forbidden_conditions(self, cs): + def _add_forbidden_conditions(self, cs: ConfigurationSpace) -> ConfigurationSpace: """ Add forbidden conditions to ensure valid configurations. Currently, Learned Entity Embedding is only valid when encoder is one hot encoder @@ -308,6 +308,10 @@ def _add_forbidden_conditions(self, cs): cs (ConfigurationSpace): Configuration space to which forbidden conditions are added. + Returns: + ConfigurationSpace: + with forbidden conditions added to the search space + """ # Learned Entity Embedding is only valid when encoder is one hot encoder diff --git a/autoPyTorch/pipeline/components/setup/network/forecasting_architecture.py b/autoPyTorch/pipeline/components/setup/network/forecasting_architecture.py index 57026728c..0f3fb9875 100644 --- a/autoPyTorch/pipeline/components/setup/network/forecasting_architecture.py +++ b/autoPyTorch/pipeline/components/setup/network/forecasting_architecture.py @@ -576,7 +576,14 @@ def forward(self, ) -> ALL_NET_OUTPUT: if isinstance(past_targets, dict): - past_targets, past_features, future_features, past_observed_targets = self._unwrap_past_targets(past_targets) + ( + past_targets, + past_features, + future_features, + past_observed_targets, + future_targets, + decoder_observed_values + ) = self._unwrap_past_targets(past_targets) x_past, x_future, x_static, loc, scale, static_context_initial_hidden, _ = self.pre_processing( past_targets=past_targets, @@ -610,13 +617,12 @@ def forward(self, def _unwrap_past_targets( self, past_targets: dict - ) -> Tuple[ - torch.Tensor, - Optional[torch.Tensor], - Optional[torch.Tensor], - Optional[torch.Tensor], - Optional[torch.BoolTensor], - Optional[torch.Tensor]]: + ) -> Tuple[torch.Tensor, + Optional[torch.Tensor], + Optional[torch.Tensor], + Optional[torch.Tensor], + Optional[torch.BoolTensor], + Optional[torch.Tensor]]: """ Time series forecasting network requires multiple inputs for the forward pass which is different to how pytorch networks usually work. SWA's update_bn in line #452 of trainer choice, does not unwrap the dictionary of the @@ -637,7 +643,14 @@ def _unwrap_past_targets( future_features = past_targets_copy.pop('future_features', None) past_observed_targets = past_targets_copy.pop('past_observed_targets', None) decoder_observed_values = past_targets_copy.pop('decoder_observed_values', None) - return past_targets,past_features,future_features,past_observed_targets + return ( + past_targets, + past_features, + future_features, + past_observed_targets, + future_targets, + decoder_observed_values + ) def pred_from_net_output(self, net_output: ALL_NET_OUTPUT) -> torch.Tensor: if self.output_type == 'regression': @@ -730,9 +743,16 @@ def forward(self, future_features: Optional[torch.Tensor] = None, past_observed_targets: Optional[torch.BoolTensor] = None, decoder_observed_values: Optional[torch.Tensor] = None, ) -> ALL_NET_OUTPUT: - + if isinstance(past_targets, dict): - past_targets, past_features, future_features, past_observed_targets = self._unwrap_past_targets(past_targets) + ( + past_targets, + past_features, + future_features, + past_observed_targets, + future_targets, + decoder_observed_values + ) = self._unwrap_past_targets(past_targets) x_past, _, x_static, loc, scale, static_context_initial_hidden, past_targets = self.pre_processing( past_targets=past_targets, @@ -1025,7 +1045,14 @@ def forward(self, decoder_observed_values: Optional[torch.Tensor] = None, ) -> ALL_NET_OUTPUT: if isinstance(past_targets, dict): - past_targets, past_features, future_features, past_observed_targets = self._unwrap_past_targets(past_targets) + ( + past_targets, + past_features, + future_features, + past_observed_targets, + future_targets, + decoder_observed_values + ) = self._unwrap_past_targets(past_targets) encode_length = min(self.window_size, past_targets.shape[1]) @@ -1295,7 +1322,14 @@ def forward(self, # type: ignore[override] Tuple[torch.Tensor, torch.Tensor]]: if isinstance(past_targets, dict): - past_targets, past_features, future_features, past_observed_targets = self._unwrap_past_targets(past_targets) + ( + past_targets, + past_features, + future_features, + past_observed_targets, + future_targets, + decoder_observed_values + ) = self._unwrap_past_targets(past_targets) # Unlike other networks, NBEATS network is required to predict both past and future targets. # Thereby, we return two tensors for backcast and forecast diff --git a/autoPyTorch/pipeline/tabular_classification.py b/autoPyTorch/pipeline/tabular_classification.py index 2e64a6944..09eb47485 100644 --- a/autoPyTorch/pipeline/tabular_classification.py +++ b/autoPyTorch/pipeline/tabular_classification.py @@ -1,4 +1,3 @@ -import copy import warnings from typing import Any, Dict, List, Optional, Tuple, Union diff --git a/autoPyTorch/pipeline/tabular_regression.py b/autoPyTorch/pipeline/tabular_regression.py index 4737bf57d..4cd67bb9f 100644 --- a/autoPyTorch/pipeline/tabular_regression.py +++ b/autoPyTorch/pipeline/tabular_regression.py @@ -1,4 +1,3 @@ -import copy import warnings from typing import Any, Dict, List, Optional, Tuple, Union diff --git a/test/test_data/test_feature_validator.py b/test/test_data/test_feature_validator.py index f3d511a79..099ee691f 100644 --- a/test/test_data/test_feature_validator.py +++ b/test/test_data/test_feature_validator.py @@ -645,7 +645,6 @@ def test_feature_validator_get_columns_to_encode_error_feat_type(input_data_feat with pytest.raises(ValueError, match=r"Expected type of features to be in .*"): validator._validate_feat_types(X) - # Null columns in the train split but not necessarily in the test split train_features = { 'A': [np.NaN, np.NaN, np.NaN], @@ -706,25 +705,3 @@ def test_feature_validator_get_columns_to_encode_error_feat_type(input_data_feat null_columns.append(column) assert null_columns == [1] - -def test_comparator(): - numerical = 'numerical' - categorical = 'categorical' - - validator = TabularFeatureValidator - - feat_type = [numerical, categorical] * 10 - ans = [categorical] * 10 + [numerical] * 10 - feat_type = sorted( - feat_type, - key=functools.cmp_to_key(validator._comparator) - ) - assert ans == feat_type - - feat_type = [numerical] * 10 + [categorical] * 10 - ans = [categorical] * 10 + [numerical] * 10 - feat_type = sorted( - feat_type, - key=functools.cmp_to_key(validator._comparator) - ) - assert ans == feat_type diff --git a/test/test_pipeline/test_tabular_classification.py b/test/test_pipeline/test_tabular_classification.py index 20b6e1c46..3e4e3bde5 100644 --- a/test/test_pipeline/test_tabular_classification.py +++ b/test/test_pipeline/test_tabular_classification.py @@ -35,7 +35,10 @@ @pytest.fixture def exclude(): - return {'feature_preprocessor': ['SelectRatesClassification', 'SelectPercentileClassification'], 'network_embedding': ['LearnedEntityEmbedding']} + return { + 'feature_preprocessor': ['SelectRatesClassification', 'SelectPercentileClassification'], + 'network_embedding': ['LearnedEntityEmbedding'] + } @pytest.mark.parametrize("fit_dictionary_tabular", ['classification_categorical_only', From afddca55795dd8d20bee472be203c1d45fe2fda3 Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Tue, 26 Jul 2022 15:54:40 +0200 Subject: [PATCH 47/50] fix silly removal of lightgbm --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 29690c18b..2a76f011a 100755 --- a/requirements.txt +++ b/requirements.txt @@ -14,5 +14,6 @@ smac>=1.2 dask distributed>=2.2.0 catboost +lightgbm flaky tabulate \ No newline at end of file From 34c704de3dd847d5ecc6144b9ca3c9f77a6560f2 Mon Sep 17 00:00:00 2001 From: Theodoros Athanasiadis Date: Fri, 12 Aug 2022 14:59:49 +0300 Subject: [PATCH 48/50] [add] documentation update in base trainer (#468) --- .../training/trainer/base_trainer.py | 235 ++++++++++++++---- 1 file changed, 181 insertions(+), 54 deletions(-) diff --git a/autoPyTorch/pipeline/components/training/trainer/base_trainer.py b/autoPyTorch/pipeline/components/training/trainer/base_trainer.py index 40f10317f..344556dd3 100644 --- a/autoPyTorch/pipeline/components/training/trainer/base_trainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/base_trainer.py @@ -45,8 +45,17 @@ def __init__(self, An object for tracking when to stop the network training. It handles epoch based criteria as well as training based criteria. - It also allows to define a 'epoch_or_time' budget type, which means, - the first of them both which is exhausted, is honored + It also allows to define a 'epoch_or_time' budget type, which means, the first of them both which is + exhausted, is honored + + Args: + budget_type (str): + Type of budget to be used when fitting the pipeline. + Possible values are 'epochs', 'runtime', or 'epoch_or_time' + max_epochs (Optional[int], default=None): + Maximum number of epochs to train the pipeline for + max_runtime (Optional[int], default=None): + Maximum number of seconds to train the pipeline for """ self.start_time = time.time() self.budget_type = budget_type @@ -54,8 +63,19 @@ def __init__(self, self.max_runtime = max_runtime def is_max_epoch_reached(self, epoch: int) -> bool: + """ + For budget type 'epoch' or 'epoch_or_time' return True if the maximum number of epochs is reached. + + Args: + epoch (int): + the current epoch - # Make None a method to run without this constrain + Returns: + bool: + True if the current epoch is larger than the maximum epochs, False otherwise. + Additionally, returns False if the run is without this constraint. + """ + # Make None a method to run without this constraint if self.max_epochs is None: return False if self.budget_type in ['epochs', 'epoch_or_time'] and epoch > self.max_epochs: @@ -63,7 +83,15 @@ def is_max_epoch_reached(self, epoch: int) -> bool: return False def is_max_time_reached(self) -> bool: - # Make None a method to run without this constrain + """ + For budget type 'runtime' or 'epoch_or_time' return True if the maximum runtime is reached. + + Returns: + bool: + True if the maximum runtime is reached, False otherwise. + Additionally, returns False if the run is without this constraint. + """ + # Make None a method to run without this constraint if self.max_runtime is None: return False elapsed_time = time.time() - self.start_time @@ -78,14 +106,22 @@ def __init__( total_parameter_count: float, trainable_parameter_count: float, optimize_metric: Optional[str] = None, - ): + ) -> None: """ A useful object to track performance per epoch. - It allows to track train, validation and test information not only for - debug, but for research purposes (Like understanding overfit). + It allows to track train, validation and test information not only for debug, but for research purposes + (Like understanding overfit). It does so by tracking a metric/loss at the end of each epoch. + + Args: + total_parameter_count (float): + the total number of parameters of the model + trainable_parameter_count (float): + only the parameters being optimized + optimize_metric (Optional[str], default=None): + name of the metric that is used to evaluate a pipeline. """ self.performance_tracker: Dict[str, Dict] = { 'start_time': {}, @@ -121,8 +157,30 @@ def add_performance(self, test_loss: Optional[float] = None, ) -> None: """ - Tracks performance information about the run, useful for - plotting individual runs + Tracks performance information about the run, useful for plotting individual runs. + + Args: + epoch (int): + the current epoch + start_time (float): + timestamp at the beginning of current epoch + end_time (float): + timestamp when gathering the information after the current epoch + train_loss (float): + the training loss + train_metrics (Dict[str, float]): + training scores for each desired metric + val_metrics (Dict[str, float]): + validation scores for each desired metric + test_metrics (Dict[str, float]): + test scores for each desired metric + val_loss (Optional[float], default=None): + the validation loss + test_loss (Optional[float], default=None): + the test loss + + Returns: + None """ self.performance_tracker['train_loss'][epoch] = train_loss self.performance_tracker['val_loss'][epoch] = val_loss @@ -134,6 +192,18 @@ def add_performance(self, self.performance_tracker['test_metrics'][epoch] = test_metrics def get_best_epoch(self, split_type: str = 'val') -> int: + """ + Get the epoch with the best metric. + + Args: + split_type (str, default=val): + Which split's metric to consider. + Possible values are 'train' or 'val + + Returns: + int: + the epoch with the best metric + """ # If we compute for optimization, prefer the performance # metric to the loss if self.optimize_metric is not None: @@ -159,6 +229,13 @@ def get_best_epoch(self, split_type: str = 'val') -> int: )) + 1 # Epochs start at 1 def get_last_epoch(self) -> int: + """ + Get the last epoch. + + Returns: + int: + the last epoch + """ if 'train_loss' not in self.performance_tracker: return 0 else: @@ -170,7 +247,8 @@ def repr_last_epoch(self) -> str: performance Returns: - str: A nice representation of the last epoch + str: + A nice representation of the last epoch """ last_epoch = len(self.performance_tracker['train_loss']) string = "\n" @@ -202,7 +280,8 @@ def is_empty(self) -> bool: Checks if the object is empty or not Returns: - bool + bool: + True if the object is empty, False otherwise """ # if train_loss is empty, we can be sure that RunSummary is empty. return not bool(self.performance_tracker['train_loss']) @@ -210,22 +289,34 @@ def is_empty(self) -> bool: class BaseTrainerComponent(autoPyTorchTrainingComponent): """ - Base class for training + Base class for training. + Args: - weighted_loss (int, default=0): In case for classification, whether to weight - the loss function according to the distribution of classes in the target - use_stochastic_weight_averaging (bool, default=True): whether to use stochastic - weight averaging. Stochastic weight averaging is a simple average of - multiple points(model parameters) along the trajectory of SGD. SWA - has been proposed in + weighted_loss (int, default=0): + In case for classification, whether to weight the loss function according to the distribution of classes + in the target + use_stochastic_weight_averaging (bool, default=True): + whether to use stochastic weight averaging. Stochastic weight averaging is a simple average of + multiple points(model parameters) along the trajectory of SGD. SWA has been proposed in [Averaging Weights Leads to Wider Optima and Better Generalization](https://arxiv.org/abs/1803.05407) - use_snapshot_ensemble (bool, default=True): whether to use snapshot - ensemble - se_lastk (int, default=3): Number of snapshots of the network to maintain - use_lookahead_optimizer (bool, default=True): whether to use lookahead - optimizer - random_state: - **lookahead_config: + use_snapshot_ensemble (bool, default=True): + whether to use snapshot ensemble + se_lastk (int, default=3): + Number of snapshots of the network to maintain + use_lookahead_optimizer (bool, default=True): + whether to use lookahead optimizer + random_state (Optional[np.random.RandomState]): + Object that contains a seed and allows for reproducible results + swa_model (Optional[torch.nn.Module], default=None): + Averaged model used for Stochastic Weight Averaging + model_snapshots (Optional[List[torch.nn.Module]], default=None): + List of model snapshots in case snapshot ensemble is used + **lookahead_config (Any): + keyword arguments for the lookahead optimizer including: + la_steps (int): + number of lookahead steps + la_alpha (float): + linear interpolation factor. 1.0 recovers the inner optimizer. """ def __init__(self, weighted_loss: int = 0, use_stochastic_weight_averaging: bool = True, @@ -336,15 +427,21 @@ def prepare( def on_epoch_start(self, X: Dict[str, Any], epoch: int) -> None: """ - Optional place holder for AutoPytorch Extensions. + Optional placeholder for AutoPytorch Extensions. + A user can define what happens on every epoch start or every epoch end. - An user can define what happens on every epoch start or every epoch end. + Args: + X (Dict[str, Any]): + Dictionary with fitted parameters. It is a message passing mechanism, in which during a transform, + a components adds relevant information so that further stages can be properly fitted + epoch (int): + the current epoch """ pass def _swa_update(self) -> None: """ - perform swa model update + Perform Stochastic Weight Averaging model update """ if self.swa_model is None: raise ValueError("SWA model cannot be none when stochastic weight averaging is enabled") @@ -354,6 +451,7 @@ def _swa_update(self) -> None: def _se_update(self, epoch: int) -> None: """ Add latest model or swa_model to model snapshot ensemble + Args: epoch (int): current epoch @@ -373,9 +471,16 @@ def _se_update(self, epoch: int) -> None: def on_epoch_end(self, X: Dict[str, Any], epoch: int) -> bool: """ - Optional place holder for AutoPytorch Extensions. - An user can define what happens on every epoch start or every epoch end. - If returns True, the training is stopped + Optional placeholder for AutoPytorch Extensions. + A user can define what happens on every epoch start or every epoch end. + If returns True, the training is stopped. + + Args: + X (Dict[str, Any]): + Dictionary with fitted parameters. It is a message passing mechanism, in which during a transform, + a components adds relevant information so that further stages can be properly fitted + epoch (int): + the current epoch """ if X['is_cyclic_scheduler']: @@ -421,12 +526,18 @@ def train_epoch(self, train_loader: torch.utils.data.DataLoader, epoch: int, Train the model for a single epoch. Args: - train_loader (torch.utils.data.DataLoader): generator of features/label - epoch (int): The current epoch used solely for tracking purposes + train_loader (torch.utils.data.DataLoader): + generator of features/label + epoch (int): + The current epoch used solely for tracking purposes + writer (Optional[SummaryWriter]): + Object to keep track of the training loss in an event file Returns: - float: training loss - Dict[str, float]: scores for each desired metric + float: + training loss + Dict[str, float]: + scores for each desired metric """ loss_sum = 0.0 @@ -482,12 +593,16 @@ def train_step(self, data: torch.Tensor, targets: torch.Tensor) -> Tuple[float, Allows to train 1 step of gradient descent, given a batch of train/labels Args: - data (torch.Tensor): input features to the network - targets (torch.Tensor): ground truth to calculate loss + data (torch.Tensor): + input features to the network + targets (torch.Tensor): + ground truth to calculate loss Returns: - torch.Tensor: The predictions of the network - float: the loss incurred in the prediction + torch.Tensor: + The predictions of the network + float: + the loss incurred in the prediction """ # prepare data = data.float().to(self.device) @@ -513,12 +628,18 @@ def evaluate(self, test_loader: torch.utils.data.DataLoader, epoch: int, Evaluate the model in both metrics and criterion Args: - test_loader (torch.utils.data.DataLoader): generator of features/label - epoch (int): the current epoch for tracking purposes + test_loader (torch.utils.data.DataLoader): + generator of features/label + epoch (int): + the current epoch for tracking purposes + writer (Optional[SummaryWriter]): + Object to keep track of the test loss in an event file Returns: - float: test loss - Dict[str, float]: scores for each desired metric + float: + test loss + Dict[str, float]: + scores for each desired metric """ self.model.eval() @@ -576,14 +697,15 @@ def get_class_weights(self, criterion: Type[torch.nn.Module], labels: Union[np.n def data_preparation(self, X: torch.Tensor, y: torch.Tensor, ) -> Tuple[torch.Tensor, Dict[str, np.ndarray]]: """ - Depending on the trainer choice, data fed to the network might be pre-processed - on a different way. That is, in standard training we provide the data to the - network as we receive it to the loader. Some regularization techniques, like mixup - alter the data. + Depending on the trainer choice, data fed to the network might be pre-processed on a different way. That is, + in standard training we provide the data to the network as we receive it to the loader. Some regularization + techniques, like mixup alter the data. Args: - X (torch.Tensor): The batch training features - y (torch.Tensor): The batch training labels + X (torch.Tensor): + The batch training features + y (torch.Tensor): + The batch training labels Returns: torch.Tensor: that processes data @@ -595,16 +717,21 @@ def data_preparation(self, X: torch.Tensor, y: torch.Tensor, def criterion_preparation(self, y_a: torch.Tensor, y_b: torch.Tensor = None, lam: float = 1.0 ) -> Callable: # type: ignore """ - Depending on the trainer choice, the criterion is not directly applied to the - traditional y_pred/y_ground_truth pairs, but rather it might have a slight transformation. + Depending on the trainer choice, the criterion is not directly applied to the traditional + y_pred/y_ground_truth pairs, but rather it might have a slight transformation. For example, in the case of mixup training, we need to account for the lambda mixup Args: - kwargs (Dict): an expanded dictionary with modifiers to the - criterion calculation + y_a (torch.Tensor): + the batch label of the first training example used in trainer + y_b (torch.Tensor, default=None): + if applicable, the batch label of the second training example used in trainer + lam (float): + trainer coefficient Returns: - Callable: a lambda function that contains the new criterion calculation recipe + Callable: + a lambda function that contains the new criterion calculation recipe """ raise NotImplementedError() From d29d11b97fa88e0c104109548c81e78b563a49f6 Mon Sep 17 00:00:00 2001 From: Ravin Kohli <13005107+ravinkohli@users.noreply.github.com> Date: Fri, 23 Sep 2022 17:39:29 +0200 Subject: [PATCH 49/50] [FIX] apply cutout for each row. (#481) * fixed cut mix * remove unnecessary comment * change all_supported_metrics --- autoPyTorch/api/base_task.py | 4 ++-- autoPyTorch/api/tabular_classification.py | 4 ++-- autoPyTorch/api/tabular_regression.py | 4 ++-- autoPyTorch/api/time_series_forecasting.py | 4 ++-- .../training/trainer/RowCutMixTrainer.py | 17 +++++++++-------- .../training/trainer/RowCutOutTrainer.py | 16 +++++++++------- 6 files changed, 26 insertions(+), 23 deletions(-) diff --git a/autoPyTorch/api/base_task.py b/autoPyTorch/api/base_task.py index 303cefc4e..8618731f5 100644 --- a/autoPyTorch/api/base_task.py +++ b/autoPyTorch/api/base_task.py @@ -978,7 +978,7 @@ def _search( smac_scenario_args: Optional[Dict[str, Any]] = None, get_smac_object_callback: Optional[Callable] = None, tae_func: Optional[Callable] = None, - all_supported_metrics: bool = True, + all_supported_metrics: bool = False, precision: int = 32, disable_file_output: Optional[List[Union[str, DisableFileOutputParameters]]] = None, load_models: bool = True, @@ -1076,7 +1076,7 @@ def _search( TargetAlgorithm to be optimised. If None, `eval_function` available in autoPyTorch/evaluation/train_evaluator is used. Must be child class of AbstractEvaluator. - all_supported_metrics (bool: default=True): + all_supported_metrics (bool: default=False): If True, all metrics supporting current task will be calculated for each pipeline and results will be available via cv_results precision (int: default=32): diff --git a/autoPyTorch/api/tabular_classification.py b/autoPyTorch/api/tabular_classification.py index facb59f99..aa6796ae2 100644 --- a/autoPyTorch/api/tabular_classification.py +++ b/autoPyTorch/api/tabular_classification.py @@ -254,7 +254,7 @@ def search( memory_limit: int = 4096, smac_scenario_args: Optional[Dict[str, Any]] = None, get_smac_object_callback: Optional[Callable] = None, - all_supported_metrics: bool = True, + all_supported_metrics: bool = False, precision: int = 32, disable_file_output: Optional[List[Union[str, DisableFileOutputParameters]]] = None, load_models: bool = True, @@ -354,7 +354,7 @@ def search( TargetAlgorithm to be optimised. If None, `eval_function` available in autoPyTorch/evaluation/train_evaluator is used. Must be child class of AbstractEvaluator. - all_supported_metrics (bool: default=True): + all_supported_metrics (bool: default=False): If True, all metrics supporting current task will be calculated for each pipeline and results will be available via cv_results precision (int: default=32): diff --git a/autoPyTorch/api/tabular_regression.py b/autoPyTorch/api/tabular_regression.py index faf36097a..d6c30aa3a 100644 --- a/autoPyTorch/api/tabular_regression.py +++ b/autoPyTorch/api/tabular_regression.py @@ -253,7 +253,7 @@ def search( memory_limit: int = 4096, smac_scenario_args: Optional[Dict[str, Any]] = None, get_smac_object_callback: Optional[Callable] = None, - all_supported_metrics: bool = True, + all_supported_metrics: bool = False, precision: int = 32, disable_file_output: Optional[List[Union[str, DisableFileOutputParameters]]] = None, load_models: bool = True, @@ -353,7 +353,7 @@ def search( TargetAlgorithm to be optimised. If None, `eval_function` available in autoPyTorch/evaluation/train_evaluator is used. Must be child class of AbstractEvaluator. - all_supported_metrics (bool: default=True): + all_supported_metrics (bool: default=False): If True, all metrics supporting current task will be calculated for each pipeline and results will be available via cv_results precision (int: default=32): diff --git a/autoPyTorch/api/time_series_forecasting.py b/autoPyTorch/api/time_series_forecasting.py index 67f6e5eaa..d564f8f47 100644 --- a/autoPyTorch/api/time_series_forecasting.py +++ b/autoPyTorch/api/time_series_forecasting.py @@ -289,7 +289,7 @@ def search( memory_limit: Optional[int] = 4096, smac_scenario_args: Optional[Dict[str, Any]] = None, get_smac_object_callback: Optional[Callable] = None, - all_supported_metrics: bool = True, + all_supported_metrics: bool = False, precision: int = 32, disable_file_output: List = [], load_models: bool = True, @@ -396,7 +396,7 @@ def search( instances, num_params, runhistory, seed and ta. This is an advanced feature. Use only if you are familiar with [SMAC](https://automl.github.io/SMAC3/master/index.html). - all_supported_metrics (bool), (default=True): if True, all + all_supported_metrics (bool), (default=False): if True, all metrics supporting current task will be calculated for each pipeline and results will be available via cv_results precision (int), (default=32): Numeric precision used when loading diff --git a/autoPyTorch/pipeline/components/training/trainer/RowCutMixTrainer.py b/autoPyTorch/pipeline/components/training/trainer/RowCutMixTrainer.py index bb4ccdb9a..149d3bd9a 100644 --- a/autoPyTorch/pipeline/components/training/trainer/RowCutMixTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/RowCutMixTrainer.py @@ -37,17 +37,18 @@ def data_preparation(self, X: np.ndarray, y: np.ndarray, if beta <= 0 or r > self.alpha: return X, {'y_a': y, 'y_b': y[shuffled_indices], 'lam': 1} - cut_column_indices = torch.as_tensor( - self.random_state.choice( - range(n_columns), - max(1, np.int32(n_columns * lam)), - replace=False, - ), - ) # Replace the values in `cut_indices` columns with # the values from `permed_indices` - X[:, cut_column_indices] = X[shuffled_indices, :][:, cut_column_indices] + for i, idx in enumerate(shuffled_indices): + cut_column_indices = torch.as_tensor( + self.random_state.choice( + range(n_columns), + max(1, np.int32(n_columns * lam)), + replace=False, + ), + ) + X[i, cut_column_indices] = X[idx, cut_column_indices] # Since we cannot cut exactly `lam x 100 %` of rows, we need to adjust the `lam` lam = 1 - (len(cut_column_indices) / n_columns) diff --git a/autoPyTorch/pipeline/components/training/trainer/RowCutOutTrainer.py b/autoPyTorch/pipeline/components/training/trainer/RowCutOutTrainer.py index 7b679976e..13511a96f 100644 --- a/autoPyTorch/pipeline/components/training/trainer/RowCutOutTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/RowCutOutTrainer.py @@ -39,15 +39,17 @@ def data_preparation(self, X: np.ndarray, y: np.ndarray, lam = 1 return X, {'y_a': y_a, 'y_b': y_b, 'lam': lam} - size: int = np.shape(X)[1] - cut_column_indices = self.random_state.choice( - range(size), - max(1, np.int32(size * self.patch_ratio)), - replace=False, - ) + n_rows, size = np.shape(X) + for i in range(n_rows): + cut_column_indices = self.random_state.choice( + range(size), + max(1, np.int32(size * self.patch_ratio)), + replace=False, + ) + X[i, cut_column_indices] = 0 + # Mask the selected features as 0 - X[:, cut_column_indices] = 0 lam = 1 y_a = y y_b = y From 873df9a12b11712318b5f66b366c29e02148e7db Mon Sep 17 00:00:00 2001 From: Ravin Kohli <13005107+ravinkohli@users.noreply.github.com> Date: Mon, 17 Oct 2022 14:35:33 +0200 Subject: [PATCH 50/50] [FIX] ROC AUC for multi class classification (#482) * fixed cut mix * remove unnecessary comment * change all_supported_metrics * fix roc_auc for multiclass * remove unnecessary code --- .../pipeline/components/setup/network/base_network.py | 5 ++--- autoPyTorch/pipeline/components/training/metrics/base.py | 2 +- autoPyTorch/pipeline/components/training/metrics/metrics.py | 2 +- autoPyTorch/pipeline/components/training/metrics/utils.py | 4 ++-- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/autoPyTorch/pipeline/components/setup/network/base_network.py b/autoPyTorch/pipeline/components/setup/network/base_network.py index 7ec872b96..0d4d3b34d 100644 --- a/autoPyTorch/pipeline/components/setup/network/base_network.py +++ b/autoPyTorch/pipeline/components/setup/network/base_network.py @@ -56,15 +56,14 @@ def fit(self, X: Dict[str, Any], y: Any = None) -> autoPyTorchTrainingComponent: self.network = torch.nn.Sequential(X['network_embedding'], X['network_backbone'], X['network_head']) + if STRING_TO_TASK_TYPES[X['dataset_properties']['task_type']] in CLASSIFICATION_TASKS: + self.network = torch.nn.Sequential(self.network, nn.Softmax(dim=1)) # Properly set the network training device if self.device is None: self.device = get_device_from_fit_dictionary(X) self.to(self.device) - if STRING_TO_TASK_TYPES[X['dataset_properties']['task_type']] in CLASSIFICATION_TASKS: - self.final_activation = nn.Softmax(dim=1) - self.is_fitted_ = True return self diff --git a/autoPyTorch/pipeline/components/training/metrics/base.py b/autoPyTorch/pipeline/components/training/metrics/base.py index 0cac3c560..4f9037cd8 100644 --- a/autoPyTorch/pipeline/components/training/metrics/base.py +++ b/autoPyTorch/pipeline/components/training/metrics/base.py @@ -173,7 +173,7 @@ def __call__( Score function applied to prediction of estimator on X. """ y_type = type_of_target(y_true) - if y_type not in ("binary", "multilabel-indicator"): + if y_type not in ("binary", "multilabel-indicator") and self.name != 'roc_auc': raise ValueError("{0} format is not supported".format(y_type)) if y_type == "binary": diff --git a/autoPyTorch/pipeline/components/training/metrics/metrics.py b/autoPyTorch/pipeline/components/training/metrics/metrics.py index 5fa60a24d..ed0c068f2 100644 --- a/autoPyTorch/pipeline/components/training/metrics/metrics.py +++ b/autoPyTorch/pipeline/components/training/metrics/metrics.py @@ -57,7 +57,7 @@ # Score functions that need decision values -roc_auc = make_metric('roc_auc', sklearn.metrics.roc_auc_score, needs_threshold=True) +roc_auc = make_metric('roc_auc', sklearn.metrics.roc_auc_score, needs_threshold=True, multi_class= 'ovo') average_precision = make_metric('average_precision', sklearn.metrics.average_precision_score, needs_threshold=True) diff --git a/autoPyTorch/pipeline/components/training/metrics/utils.py b/autoPyTorch/pipeline/components/training/metrics/utils.py index e72c1afce..2a4865aa5 100644 --- a/autoPyTorch/pipeline/components/training/metrics/utils.py +++ b/autoPyTorch/pipeline/components/training/metrics/utils.py @@ -99,8 +99,8 @@ def get_metrics(dataset_properties: Dict[str, Any], if names is not None: for name in names: if name not in supported_metrics.keys(): - raise ValueError("Invalid name entered for task {}, currently " - "supported metrics for task include {}".format(dataset_properties['task_type'], + raise ValueError("Invalid name {} entered for task {}, currently " + "supported metrics for task include {}".format(name, dataset_properties['task_type'], list(supported_metrics.keys()))) else: metric = supported_metrics[name]