From d57b6b41402a30de27d0e0d81e3bf17de4f728a8 Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Sat, 20 Mar 2021 12:27:32 +0100 Subject: [PATCH 01/10] In progress, add_hyperparameter --- autoPyTorch/pipeline/base_pipeline.py | 2 +- .../pipeline/components/base_choice.py | 25 ++--- .../pipeline/components/base_component.py | 24 ++-- .../feature_preprocessing/KernelPCA.py | 69 +++++++----- .../feature_preprocessing/Nystroem.py | 67 +++++++---- .../PolynomialFeatures.py | 32 +++--- .../feature_preprocessing/PowerTransformer.py | 11 +- .../RandomKitchenSinks.py | 44 ++++---- .../feature_preprocessing/TruncatedSVD.py | 24 ++-- .../base_feature_preprocessor_choice.py | 2 +- .../imputation/SimpleImputer.py | 33 +++--- .../scaling/Normalizer.py | 15 ++- .../setup/augmentation/image/GaussianBlur.py | 27 +++-- .../setup/augmentation/image/GaussianNoise.py | 17 ++- .../setup/augmentation/image/RandomAffine.py | 49 ++++---- .../setup/augmentation/image/RandomCutout.py | 20 ++-- .../setup/augmentation/image/Resize.py | 13 ++- .../augmentation/image/ZeroPadAndCrop.py | 11 +- .../setup/lr_scheduler/CosineAnnealingLR.py | 19 ++-- .../CosineAnnealingWarmRestarts.py | 25 +++-- .../components/setup/lr_scheduler/CyclicLR.py | 47 +++++--- .../setup/lr_scheduler/ExponentialLR.py | 17 +-- .../setup/lr_scheduler/ReduceLROnPlateau.py | 35 ++++-- .../components/setup/lr_scheduler/StepLR.py | 26 +++-- .../network_backbone/ConvNetImageBackbone.py | 88 +++++++-------- .../network_backbone/DenseNetImageBackone.py | 72 ++++++------ autoPyTorch/utils/common.py | 105 +++++++++++++++++- .../hyperparameter_search_space_update.py | 25 +++-- 28 files changed, 578 insertions(+), 366 deletions(-) diff --git a/autoPyTorch/pipeline/base_pipeline.py b/autoPyTorch/pipeline/base_pipeline.py index 2e70478a3..30d63519f 100644 --- a/autoPyTorch/pipeline/base_pipeline.py +++ b/autoPyTorch/pipeline/base_pipeline.py @@ -365,7 +365,7 @@ def _get_base_search_space( cs.add_configuration_space( node_name, node.get_hyperparameter_search_space(dataset_properties, # type: ignore[arg-type] - **node._get_search_space_updates()), + **node._get_search_space_updates(node_name)), ) # If the node is a choice, we have to figure out which of its # choices are actually legal choices diff --git a/autoPyTorch/pipeline/components/base_choice.py b/autoPyTorch/pipeline/components/base_choice.py index b6fffbaf2..d222cc6d2 100644 --- a/autoPyTorch/pipeline/components/base_choice.py +++ b/autoPyTorch/pipeline/components/base_choice.py @@ -9,7 +9,8 @@ from sklearn.utils import check_random_state from autoPyTorch.pipeline.components.base_component import autoPyTorchComponent -from autoPyTorch.utils.common import FitRequirement +from autoPyTorch.utils.common import FitRequirement, HyperparameterSearchSpace +from autoPyTorch.utils.hyperparameter_search_space_update import HyperparameterSearchSpaceUpdate class autoPyTorchChoice(object): @@ -49,7 +50,7 @@ def __init__(self, # self.set_hyperparameters(self.configuration) self.choice: Optional[autoPyTorchComponent] = None - self._cs_updates: Dict[str, Tuple] = dict() + self._cs_updates: Dict[str, HyperparameterSearchSpaceUpdate] = dict() def get_fit_requirements(self) -> Optional[List[FitRequirement]]: if self.choice is not None: @@ -247,8 +248,7 @@ def _check_dataset_properties(self, dataset_properties: Dict[str, Any]) -> None: """ assert isinstance(dataset_properties, dict), "dataset_properties must be a dictionary" - def _apply_search_space_update(self, name: str, new_value_range: Union[List, Tuple], - default_value: Union[int, float, str], log: bool = False) -> None: + def _apply_search_space_update(self, hyperparameter_search_space_update: HyperparameterSearchSpaceUpdate) -> None: """Allows the user to update a hyperparameter Arguments: @@ -257,11 +257,9 @@ def _apply_search_space_update(self, name: str, new_value_range: Union[List, Tup log {bool} -- is hyperparameter logscale """ - if len(new_value_range) == 0: - raise ValueError("The new value range needs at least one value") - self._cs_updates[name] = tuple([new_value_range, default_value, log]) + self._cs_updates[hyperparameter_search_space_update.hyperparameter] = hyperparameter_search_space_update - def _get_search_space_updates(self, prefix: Optional[str] = None) -> Dict[str, Tuple]: + def _get_search_space_updates(self, hyperparameter: str) -> Dict[str, HyperparameterSearchSpace]: """Get the search space updates with the given prefix Keyword Arguments: @@ -270,12 +268,11 @@ def _get_search_space_updates(self, prefix: Optional[str] = None) -> Dict[str, T Returns: dict -- Mapping of search space updates. Keys don't contain the prefix. """ - if prefix is None: - return self._cs_updates - result: Dict[str, Tuple] = dict() - # iterate over all search space updates of this node and filter the ones out, that have the given prefix + result: Dict[str, HyperparameterSearchSpace] = dict() + + # iterate over all search space updates of this node and keep the ones that have the given prefix for key in self._cs_updates.keys(): - if key.startswith(prefix): - result[key[len(prefix) + 1:]] = self._cs_updates[key] + if hyperparameter == key: + result[hyperparameter] = self._cs_updates[hyperparameter].get_search_space() return result diff --git a/autoPyTorch/pipeline/components/base_component.py b/autoPyTorch/pipeline/components/base_component.py index 80fb1ddd0..81845a6f5 100644 --- a/autoPyTorch/pipeline/components/base_component.py +++ b/autoPyTorch/pipeline/components/base_component.py @@ -10,7 +10,8 @@ from sklearn.base import BaseEstimator -from autoPyTorch.utils.common import FitRequirement +from autoPyTorch.utils.common import FitRequirement, HyperparameterSearchSpace +from autoPyTorch.utils.hyperparameter_search_space_update import HyperparameterSearchSpaceUpdate def find_components( @@ -96,7 +97,7 @@ class autoPyTorchComponent(BaseEstimator): def __init__(self) -> None: super().__init__() self._fit_requirements: List[FitRequirement] = list() - self._cs_updates: Dict[str, Tuple] = dict() + self._cs_updates: Dict[str, HyperparameterSearchSpaceUpdate] = dict() @classmethod def get_required_properties(cls) -> Optional[List[str]]: @@ -253,8 +254,7 @@ def __str__(self) -> str: name = self.get_properties()['name'] return "autoPyTorch.pipeline %s" % name - def _apply_search_space_update(self, name: str, new_value_range: Union[List, Tuple], - default_value: Union[int, float, str], log: bool = False) -> None: + def _apply_search_space_update(self, hyperparameter_search_space_update: HyperparameterSearchSpaceUpdate) -> None: """Allows the user to update a hyperparameter Arguments: @@ -263,11 +263,9 @@ def _apply_search_space_update(self, name: str, new_value_range: Union[List, Tup log {bool} -- is hyperparameter logscale """ - if len(new_value_range) == 0: - raise ValueError("The new value range needs at least one value") - self._cs_updates[name] = tuple([new_value_range, default_value, log]) + self._cs_updates[hyperparameter_search_space_update.hyperparameter] = hyperparameter_search_space_update - def _get_search_space_updates(self, prefix: Optional[str] = None) -> Dict[str, Tuple]: + def _get_search_space_updates(self, hyperparameter: str) -> Dict[str, HyperparameterSearchSpace]: """Get the search space updates with the given prefix Keyword Arguments: @@ -276,13 +274,11 @@ def _get_search_space_updates(self, prefix: Optional[str] = None) -> Dict[str, T Returns: dict -- Mapping of search space updates. Keys don't contain the prefix. """ - if prefix is None: - return self._cs_updates - result: Dict[str, Tuple] = dict() + + result: Dict[str, HyperparameterSearchSpace] = dict() # iterate over all search space updates of this node and keep the ones that have the given prefix for key in self._cs_updates.keys(): - if key.startswith(prefix): - # different for autopytorch component as the hyperparameter - result[key[len(prefix):]] = self._cs_updates[key] + if hyperparameter == key: + result[hyperparameter] = self._cs_updates[hyperparameter].get_search_space() return result diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/KernelPCA.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/KernelPCA.py index 4fb77b90f..b9a370429 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/KernelPCA.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/KernelPCA.py @@ -14,9 +14,9 @@ import sklearn.decomposition from sklearn.base import BaseEstimator -from autoPyTorch.pipeline.components.preprocessing.tabular_preprocessing.feature_preprocessing.\ +from autoPyTorch.pipeline.components.preprocessing.tabular_preprocessing.feature_preprocessing. \ base_feature_preprocessor import autoPyTorchFeaturePreprocessingComponent -from autoPyTorch.utils.common import FitRequirement +from autoPyTorch.utils.common import FitRequirement, HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter class KernelPCA(autoPyTorchFeaturePreprocessingComponent): @@ -48,36 +48,55 @@ def fit(self, X: Dict[str, Any], y: Any = None) -> BaseEstimator: @staticmethod def get_hyperparameter_search_space( dataset_properties: Optional[Dict[str, str]] = None, - n_components: Tuple[Tuple, float] = ((0.5, 0.9), 0.5), - kernel: Tuple[Tuple, str] = (('poly', 'rbf', 'sigmoid', 'cosine'), 'rbf'), - gamma: Tuple[Tuple, float, bool] = ((3.0517578125e-05, 8), 0.01, True), - degree: Tuple[Tuple, int] = ((2, 5), 3), - coef0: Tuple[Tuple, float] = ((-1, 1), 0) + n_components: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='n_components', + value_range=(0.5, 0.9), + default_value=0.5, + log=False), + kernel: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='kernel', + value_range=('poly', 'rbf', 'sigmoid', 'cosine'), + default_value='rbf', + log=False), + gamma: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='gamma', + value_range=(3.0517578125e-05, 8), + default_value=0.01, + log=True), + degree: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='degree', + value_range=(2, 5), + default_value=3, + log=True), + coef0: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='coef0', + value_range=(-1, 1), + default_value=0, + log=False) ) -> ConfigurationSpace: + cs = ConfigurationSpace() + if dataset_properties is not None: n_features = len(dataset_properties['numerical_columns']) - n_components = ((floor(n_components[0][0] * n_features), ceil(n_components[0][1] * n_features)), - ceil(n_components[1] * n_features)) + if n_features == 1: + log = False + else: + log = n_components.log + n_components = HyperparameterSearchSpace(hyperparameter='n_components', + value_range=(floor(n_components.value_range[0] * n_features), + ceil(n_components.value_range[1] * n_features)), + default_value=ceil(n_components.default_value * n_features), + log=log) else: - n_components = ((10, 2000), 100) - - n_components = UniformIntegerHyperparameter( - "n_components", lower=n_components[0][0], upper=n_components[0][1], default_value=n_components[1]) - kernel_hp = CategoricalHyperparameter('kernel', choices=kernel[0], default_value=kernel[1]) - gamma = UniformFloatHyperparameter( - "gamma", - lower=gamma[0][0], upper=gamma[0][1], - log=gamma[2], - default_value=gamma[1], - ) - coef0 = UniformFloatHyperparameter("coef0", lower=coef0[0][0], upper=coef0[0][1], default_value=coef0[1]) - cs = ConfigurationSpace() - cs.add_hyperparameters([n_components, kernel_hp, gamma, coef0]) + n_components = HyperparameterSearchSpace(hyperparameter='n_components', + value_range=(10, 2000), + default_value=100, + log=n_components.log) + + add_hyperparameter(cs, n_components, UniformIntegerHyperparameter) + kernel_hp = get_hyperparameter(kernel, CategoricalHyperparameter) + gamma = get_hyperparameter(gamma, UniformFloatHyperparameter) + coef0 = get_hyperparameter(coef0, UniformFloatHyperparameter) + cs.add_hyperparameters([kernel_hp, gamma, coef0]) if "poly" in kernel_hp.choices: - degree = UniformIntegerHyperparameter('degree', lower=degree[0][0], upper=degree[0][1], - default_value=degree[1]) + degree = get_hyperparameter(degree, UniformIntegerHyperparameter) cs.add_hyperparameters([degree]) degree_depends_on_poly = EqualsCondition(degree, kernel_hp, "poly") cs.add_conditions([degree_depends_on_poly]) diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/Nystroem.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/Nystroem.py index 93be983e9..d73aac3a9 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/Nystroem.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/Nystroem.py @@ -16,6 +16,7 @@ from autoPyTorch.pipeline.components.preprocessing.tabular_preprocessing.feature_preprocessing.\ base_feature_preprocessor import autoPyTorchFeaturePreprocessingComponent +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter class Nystroem(autoPyTorchFeaturePreprocessingComponent): @@ -44,13 +45,30 @@ def fit(self, X: Dict[str, Any], y: Any = None) -> BaseEstimator: @staticmethod def get_hyperparameter_search_space( dataset_properties: Optional[Dict[str, str]] = None, - n_components: Tuple[Tuple, float, bool] = ((0.5, 0.9), 0.5, True), - kernel: Tuple[Tuple, str] = (('poly', 'rbf', 'sigmoid', 'cosine'), 'rbf'), - gamma: Tuple[Tuple, float, bool] = ((3.0517578125e-05, 8), 0.01, True), - degree: Tuple[Tuple, int] = ((2, 5), 3), - coef0: Tuple[Tuple, float] = ((-1, 1), 0) + n_components: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='n_components', + value_range=(0.5, 0.9), + default_value=0.5, + log=False), + kernel: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='kernel', + value_range=('poly', 'rbf', 'sigmoid', 'cosine'), + default_value='rbf', + log=False), + gamma: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='gamma', + value_range=(3.0517578125e-05, 8), + default_value=0.01, + log=True), + degree: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='degree', + value_range=(2, 5), + default_value=3, + log=True), + coef0: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='coef0', + value_range=(-1, 1), + default_value=0, + log=False) ) -> ConfigurationSpace: + cs = ConfigurationSpace() + if dataset_properties is not None: n_features = len(dataset_properties['numerical_columns']) # if numerical features are 1, set log to False @@ -58,27 +76,28 @@ def get_hyperparameter_search_space( log = False else: log = n_components[2] - n_components = ((floor(n_components[0][0] * n_features), ceil(n_components[0][1] * n_features)), - ceil(n_components[1] * n_features), log) + n_components = HyperparameterSearchSpace(hyperparameter='n_components', + value_range=(floor(n_components.value_range[0] * n_features), + ceil(n_components.value_range[1] * n_features)), + default_value=ceil(n_components.default_value * n_features), + log=log) else: - n_components = ((10, 2000), 100, True) + n_components = HyperparameterSearchSpace(hyperparameter='n_components', + value_range=(10, 2000), + default_value=100, + log=n_components.log) - n_components = UniformIntegerHyperparameter( - "n_components", lower=n_components[0][0], upper=n_components[0][1], - default_value=n_components[1], log=n_components[2]) - kernel_hp = CategoricalHyperparameter('kernel', choices=kernel[0], default_value=kernel[1]) - gamma = UniformFloatHyperparameter( - "gamma", - lower=gamma[0][0], upper=gamma[0][1], - log=gamma[2], - default_value=gamma[1], - ) - degree = UniformIntegerHyperparameter('degree', lower=degree[0][0], upper=degree[0][1], default_value=degree[1]) - coef0 = UniformFloatHyperparameter("coef0", lower=coef0[0][0], upper=coef0[0][1], default_value=coef0[1]) - cs = ConfigurationSpace() - cs.add_hyperparameters([n_components, kernel_hp, degree, gamma, coef0]) + add_hyperparameter(cs, n_components, UniformIntegerHyperparameter) + kernel_hp = get_hyperparameter(kernel, CategoricalHyperparameter) + gamma = get_hyperparameter(gamma, UniformFloatHyperparameter) + coef0 = get_hyperparameter(coef0, UniformFloatHyperparameter) + cs.add_hyperparameters([kernel_hp, gamma, coef0]) - degree_depends_on_poly = EqualsCondition(degree, kernel_hp, "poly") + if "poly" in kernel_hp.choices: + degree = get_hyperparameter(degree, UniformIntegerHyperparameter) + cs.add_hyperparameters([degree]) + degree_depends_on_poly = EqualsCondition(degree, kernel_hp, "poly") + cs.add_conditions([degree_depends_on_poly]) kernels = [] if "sigmoid" in kernel_hp.choices: kernels.append("sigmoid") @@ -91,7 +110,7 @@ def get_hyperparameter_search_space( if "poly" in kernel_hp.choices: kernels.append("poly") gamma_condition = InCondition(gamma, kernel_hp, kernels) - cs.add_conditions([degree_depends_on_poly, coef0_condition, gamma_condition]) + cs.add_conditions([coef0_condition, gamma_condition]) return cs @staticmethod diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/PolynomialFeatures.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/PolynomialFeatures.py index 9f542acd0..2a51ecbbf 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/PolynomialFeatures.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/PolynomialFeatures.py @@ -11,8 +11,9 @@ import sklearn.decomposition from sklearn.base import BaseEstimator -from autoPyTorch.pipeline.components.preprocessing.tabular_preprocessing.feature_preprocessing.\ +from autoPyTorch.pipeline.components.preprocessing.tabular_preprocessing.feature_preprocessing. \ base_feature_preprocessor import autoPyTorchFeaturePreprocessingComponent +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter class PolynomialFeatures(autoPyTorchFeaturePreprocessingComponent): @@ -27,7 +28,6 @@ def __init__(self, degree: int = 2, interaction_only: bool = False, super().__init__() def fit(self, X: Dict[str, Any], y: Any = None) -> BaseEstimator: - self.preprocessor['numerical'] = sklearn.preprocessing.PolynomialFeatures( degree=self.degree, interaction_only=self.interaction_only, include_bias=self.include_bias) @@ -42,20 +42,22 @@ def get_properties(dataset_properties: Optional[Dict[str, str]] = None) -> Dict[ @staticmethod def get_hyperparameter_search_space( dataset_properties: Optional[Dict[str, str]] = None, - degree: Tuple[Tuple, int] = ((2, 3), 2), - interaction_only: Tuple[Tuple, bool] = ((True, False), False), - include_bias: Tuple[Tuple, bool] = ((True, False), False) + degree: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='degree', + value_range=(2, 3), + default_value=2, + log=True), + interaction_only: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='interaction_only', + value_range=(True, False), + default_value=False, + log=False), + include_bias: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='include_bias', + value_range=(True, False), + default_value=False, + log=False), ) -> ConfigurationSpace: - - degree = UniformIntegerHyperparameter("degree", lower=degree[0][0], upper=degree[0][1], default_value=degree[1]) - interaction_only = CategoricalHyperparameter("interaction_only", - choices=interaction_only[0], - default_value=interaction_only[1]) - include_bias = CategoricalHyperparameter("include_bias", - choices=include_bias[0], - default_value=include_bias[1]) - cs = ConfigurationSpace() - cs.add_hyperparameters([degree, interaction_only, include_bias]) + add_hyperparameter(cs, degree, UniformIntegerHyperparameter) + add_hyperparameter(cs, interaction_only, CategoricalHyperparameter) + add_hyperparameter(cs, include_bias, CategoricalHyperparameter) return cs diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/PowerTransformer.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/PowerTransformer.py index c02606c3d..95498d1be 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/PowerTransformer.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/PowerTransformer.py @@ -12,6 +12,7 @@ from autoPyTorch.pipeline.components.preprocessing.tabular_preprocessing.feature_preprocessing. \ base_feature_preprocessor import autoPyTorchFeaturePreprocessingComponent +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter class PowerTransformer(autoPyTorchFeaturePreprocessingComponent): @@ -37,13 +38,13 @@ def get_properties(dataset_properties: Optional[Dict[str, str]] = None) -> Dict[ @staticmethod def get_hyperparameter_search_space( dataset_properties: Optional[Dict[str, str]] = None, - standardize: Tuple[Tuple, bool] = ((True, False), True) + standardize: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='standardize', + value_range=(True, False), + default_value=True, + log=False), ) -> ConfigurationSpace: - standardize = CategoricalHyperparameter("standardize", - choices=standardize[0], - default_value=standardize[1]) cs = ConfigurationSpace() - cs.add_hyperparameters([standardize]) + add_hyperparameter(cs, standardize, CategoricalHyperparameter) return cs diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/RandomKitchenSinks.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/RandomKitchenSinks.py index 8f03e1880..c7f60da4a 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/RandomKitchenSinks.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/RandomKitchenSinks.py @@ -14,7 +14,7 @@ from autoPyTorch.pipeline.components.preprocessing.tabular_preprocessing.feature_preprocessing. \ base_feature_preprocessor import autoPyTorchFeaturePreprocessingComponent - +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter class RandomKitchenSinks(autoPyTorchFeaturePreprocessingComponent): def __init__(self, n_components: int = 100, @@ -35,35 +35,37 @@ def fit(self, X: Dict[str, Any], y: Any = None) -> BaseEstimator: @staticmethod def get_hyperparameter_search_space( dataset_properties: Optional[Dict[str, str]] = None, - n_components: Tuple[Tuple, float, bool] = ((0.5, 0.9), 0.5, True), - gamma: Tuple[Tuple, float, bool] = ((3.0517578125e-05, 8), 1.0, True), - degree: Tuple[Tuple, int] = ((2, 5), 3), - coef0: Tuple[Tuple, float] = ((-1, 1), 0) + n_components: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='n_components', + value_range=(0.5, 0.9), + default_value=0.5, + log=False), + gamma: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='gamma', + value_range=(3.0517578125e-05, 8), + default_value=0.01, + log=True), ) -> ConfigurationSpace: + cs = ConfigurationSpace() if dataset_properties is not None: n_features = len(dataset_properties['numerical_columns']) - # if numerical features are 1, set log to False if n_features == 1: log = False else: - log = n_components[2] - n_components = ((floor(n_components[0][0] * n_features), ceil(n_components[0][1] * n_features)), - ceil(n_components[1] * n_features), log) + log = n_components.log + n_components = HyperparameterSearchSpace(hyperparameter='n_components', + value_range=(floor(n_components.value_range[0] * n_features), + ceil(n_components.value_range[1] * n_features)), + default_value=ceil(n_components.default_value * n_features), + log=log) else: - n_components = ((10, 2000), 100, True) + n_components = HyperparameterSearchSpace(hyperparameter='n_components', + value_range=(10, 2000), + default_value=100, + log=n_components.log) - n_components = UniformIntegerHyperparameter( - "n_components", lower=n_components[0][0], upper=n_components[0][1], - default_value=n_components[1], log=n_components[2]) - gamma = UniformFloatHyperparameter( - "gamma", - lower=gamma[0][0], upper=gamma[0][1], - log=gamma[2], - default_value=gamma[1], - ) - cs = ConfigurationSpace() - cs.add_hyperparameters([n_components, gamma]) + add_hyperparameter(cs, n_components, UniformIntegerHyperparameter) + + add_hyperparameter(cs, gamma, UniformFloatHyperparameter) return cs @staticmethod diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/TruncatedSVD.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/TruncatedSVD.py index 558fdb4de..41c74d269 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/TruncatedSVD.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/TruncatedSVD.py @@ -11,8 +11,9 @@ import sklearn.decomposition from sklearn.base import BaseEstimator -from autoPyTorch.pipeline.components.preprocessing.tabular_preprocessing.feature_preprocessing\ +from autoPyTorch.pipeline.components.preprocessing.tabular_preprocessing.feature_preprocessing \ .base_feature_preprocessor import autoPyTorchFeaturePreprocessingComponent +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter class TruncatedSVD(autoPyTorchFeaturePreprocessingComponent): @@ -38,18 +39,25 @@ def get_properties(dataset_properties: Optional[Dict[str, str]] = None) -> Dict[ @staticmethod def get_hyperparameter_search_space( dataset_properties: Optional[Dict[str, str]] = None, - target_dim: Tuple[Tuple, float] = ((0.5, 0.9), 0.5), + target_dim: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='target_dim', + value_range=(0.5, 0.9), + default_value=0.5, + log=False), ) -> ConfigurationSpace: cs = ConfigurationSpace() if dataset_properties is not None: n_features = len(dataset_properties['numerical_columns']) - target_dim = ((floor(target_dim[0][0] * n_features), floor(target_dim[0][1] * n_features)), - floor(target_dim[1] * n_features)) + target_dim = HyperparameterSearchSpace(hyperparameter=target_dim.hyperparameter, + value_range=(floor(target_dim.value_range[0] * n_features), + floor(target_dim.value_range[1] * n_features)), + default_value=floor(target_dim[1] * n_features), + log=target_dim.log) else: - target_dim = ((10, 256), 128) - target_dim = UniformIntegerHyperparameter("target_dim", lower=target_dim[0][0], - upper=target_dim[0][1], default_value=target_dim[1]) - cs.add_hyperparameters([target_dim]) + target_dim = HyperparameterSearchSpace(hyperparameter=target_dim.hyperparameter, + value_range=(10, 256), + default_value=128, + log=target_dim.log) + add_hyperparameter(cs, target_dim, UniformIntegerHyperparameter) return cs diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/base_feature_preprocessor_choice.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/base_feature_preprocessor_choice.py index 56af71877..b8b17e38d 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/base_feature_preprocessor_choice.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/base_feature_preprocessor_choice.py @@ -104,7 +104,7 @@ def get_hyperparameter_search_space(self, # add only child hyperparameters of early_preprocessor choices for name in preprocessor.choices: - updates = self._get_search_space_updates(prefix=name) + updates = self._get_search_space_updates(hyperparameter=name) config_space = available_[name].get_hyperparameter_search_space(dataset_properties, # type:ignore **updates) parent_hyperparameter = {'parent': preprocessor, 'value': name} diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/imputation/SimpleImputer.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/imputation/SimpleImputer.py index 6b2a81bc9..5b933eac2 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/imputation/SimpleImputer.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/imputation/SimpleImputer.py @@ -10,7 +10,7 @@ from sklearn.impute import SimpleImputer as SklearnSimpleImputer from autoPyTorch.pipeline.components.preprocessing.tabular_preprocessing.imputation.base_imputer import BaseImputer - +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter class SimpleImputer(BaseImputer): """ @@ -65,28 +65,29 @@ def fit(self, X: Dict[str, Any], y: Any = None) -> BaseImputer: return self @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict[str, Any]] = None, - numerical_strategy: Tuple[Tuple, str] = (("mean", "median", - "most_frequent", "constant_zero"), - "mean"), - categorical_strategy: Tuple[Tuple, str] = (("most_frequent", - "constant_!missing!"), - "most_frequent") + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict[str, Any]] = None, + numerical_strategy: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='numerical_strategy', + value_range=("mean", "median", + "most_frequent", + "constant_zero"), + default_value="mean", + log=False), + categorical_strategy: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='numerical_strategy', + value_range=("most_frequent", + "constant_!missing!"), + default_value="most_frequent", + log=False) ) -> ConfigurationSpace: cs = ConfigurationSpace() assert dataset_properties is not None, "To create hyperparameter search space" \ ", dataset_properties should not be None" if len(dataset_properties['numerical_columns']) != 0: - numerical_strategy = CategoricalHyperparameter("numerical_strategy", - numerical_strategy[0], - default_value=numerical_strategy[1]) - cs.add_hyperparameter(numerical_strategy) + add_hyperparameter(cs, numerical_strategy, CategoricalHyperparameter) if len(dataset_properties['categorical_columns']) != 0: - categorical_strategy = CategoricalHyperparameter("categorical_strategy", - categorical_strategy[0], - default_value=categorical_strategy[1]) - cs.add_hyperparameter(categorical_strategy) + add_hyperparameter(cs, categorical_strategy, CategoricalHyperparameter) + return cs @staticmethod diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/Normalizer.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/Normalizer.py index f0e6dc0ff..acdf6ac64 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/Normalizer.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/Normalizer.py @@ -10,6 +10,7 @@ from sklearn.preprocessing import Normalizer as SklearnNormalizer from autoPyTorch.pipeline.components.preprocessing.tabular_preprocessing.scaling.base_scaler import BaseScaler +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter class Normalizer(BaseScaler): @@ -36,13 +37,15 @@ def fit(self, X: Dict[str, Any], y: Any = None) -> BaseScaler: return self @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict[str, Any]] = None, - norm: Tuple[Tuple, str] = (("mean_abs", "mean_squared", "max"), - "mean_squared") - ) -> ConfigurationSpace: + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict[str, Any]] = None, + norm: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="norm", + value_range=("mean_abs", "mean_squared", "max"), + default_value="mean_squared", + log=False) + ) -> ConfigurationSpace: cs = ConfigurationSpace() - norm = CategoricalHyperparameter("norm", norm[0], default_value=norm[1]) - cs.add_hyperparameter(norm) + add_hyperparameter(cs, norm, CategoricalHyperparameter) return cs @staticmethod diff --git a/autoPyTorch/pipeline/components/setup/augmentation/image/GaussianBlur.py b/autoPyTorch/pipeline/components/setup/augmentation/image/GaussianBlur.py index e977b0633..8586ff47f 100644 --- a/autoPyTorch/pipeline/components/setup/augmentation/image/GaussianBlur.py +++ b/autoPyTorch/pipeline/components/setup/augmentation/image/GaussianBlur.py @@ -13,6 +13,7 @@ import numpy as np from autoPyTorch.pipeline.components.setup.augmentation.image.base_image_augmenter import BaseImageAugmenter +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter class GaussianBlur(BaseImageAugmenter): @@ -31,20 +32,24 @@ def fit(self, X: Dict[str, Any], y: Any = None) -> BaseImageAugmenter: @staticmethod def get_hyperparameter_search_space( dataset_properties: Optional[Dict[str, str]] = None, - use_augmenter: Tuple[Tuple, bool] = ((True, False), True), - sigma_min: Tuple[Tuple, int] = ((0, 3), 0), - sigma_offset: Tuple[Tuple, float] = ((0.0, 3.0), 0.5), + use_augmenter: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_augmenter", + value_range=(True, False), + default_value=True, + log=False), + sigma_min: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="sigma_min", + value_range=(0, 3), + default_value=0, + log=False), + sigma_offset: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="sigma_min", + value_range=(0.0, 3.0), + default_value=0.5, + log=False), ) -> ConfigurationSpace: - cs = ConfigurationSpace() - use_augmenter = CategoricalHyperparameter('use_augmenter', choices=use_augmenter[0], - default_value=use_augmenter[1]) - sigma_min = UniformFloatHyperparameter('sigma_min', lower=sigma_min[0][0], upper=sigma_min[0][1], - default_value=0) - sigma_offset = UniformFloatHyperparameter('sigma_offset', lower=sigma_offset[0][0], upper=sigma_offset[0][1], - default_value=0.5) + use_augmenter = get_hyperparameter(use_augmenter, CategoricalHyperparameter) + sigma_min = get_hyperparameter(sigma_min, UniformFloatHyperparameter) + sigma_offset = get_hyperparameter(sigma_offset, UniformFloatHyperparameter) cs.add_hyperparameters([use_augmenter, sigma_min, sigma_offset]) - # only add hyperparameters to configuration space if we are using the augmenter cs.add_condition(CS.EqualsCondition(sigma_min, use_augmenter, True)) cs.add_condition(CS.EqualsCondition(sigma_offset, use_augmenter, True)) diff --git a/autoPyTorch/pipeline/components/setup/augmentation/image/GaussianNoise.py b/autoPyTorch/pipeline/components/setup/augmentation/image/GaussianNoise.py index 064c564cd..e3fd399f3 100644 --- a/autoPyTorch/pipeline/components/setup/augmentation/image/GaussianNoise.py +++ b/autoPyTorch/pipeline/components/setup/augmentation/image/GaussianNoise.py @@ -13,6 +13,7 @@ import numpy as np from autoPyTorch.pipeline.components.setup.augmentation.image.base_image_augmenter import BaseImageAugmenter +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter class GaussianNoise(BaseImageAugmenter): @@ -30,15 +31,19 @@ def fit(self, X: Dict[str, Any], y: Any = None) -> BaseImageAugmenter: @staticmethod def get_hyperparameter_search_space( dataset_properties: Optional[Dict[str, str]] = None, - use_augmenter: Tuple[Tuple[bool, bool], bool] = ((True, False), True), - sigma_offset: Tuple[Tuple[float, float], float] = ((0.0, 3.0), 0.3) + use_augmenter: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_augmenter", + value_range=(True, False), + default_value=True, + log=False), + sigma_offset: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="sigma_offset", + value_range=(0.0, 3.0), + default_value=0.3, + log=False), ) -> ConfigurationSpace: cs = ConfigurationSpace() - sigma_offset = UniformFloatHyperparameter('sigma_offset', lower=sigma_offset[0][0], upper=sigma_offset[0][1], - default_value=sigma_offset[1]) - use_augmenter = CategoricalHyperparameter('use_augmenter', choices=use_augmenter[0], - default_value=use_augmenter[1]) + use_augmenter = get_hyperparameter(use_augmenter, CategoricalHyperparameter) + sigma_offset = get_hyperparameter(sigma_offset, UniformFloatHyperparameter) cs.add_hyperparameters([use_augmenter, sigma_offset]) # only add hyperparameters to configuration space if we are using the augmenter cs.add_condition(CS.EqualsCondition(sigma_offset, use_augmenter, True)) diff --git a/autoPyTorch/pipeline/components/setup/augmentation/image/RandomAffine.py b/autoPyTorch/pipeline/components/setup/augmentation/image/RandomAffine.py index a9797a6e5..4fe5757ad 100644 --- a/autoPyTorch/pipeline/components/setup/augmentation/image/RandomAffine.py +++ b/autoPyTorch/pipeline/components/setup/augmentation/image/RandomAffine.py @@ -14,6 +14,7 @@ import numpy as np from autoPyTorch.pipeline.components.setup.augmentation.image.base_image_augmenter import BaseImageAugmenter +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter class RandomAffine(BaseImageAugmenter): @@ -38,29 +39,37 @@ def fit(self, X: Dict[str, Any], y: Any = None) -> BaseImageAugmenter: @staticmethod def get_hyperparameter_search_space( dataset_properties: Optional[Dict[str, str]] = None, - use_augmenter: Tuple[Tuple, bool] = ((True, False), True), - scale_offset: Tuple[Tuple, float] = ((0, 0.4), 0.2), - translate_percent_offset: Tuple[Tuple, float] = ((0, 0.4), 0.2), - shear: Tuple[Tuple, int] = ((0, 45), 30), - rotate: Tuple[Tuple, int] = ((0, 360), 45) + use_augmenter: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_augmenter", + value_range=(True, False), + default_value=True, + log=False), + scale_offset: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="scale_offset", + value_range=(0, 0.4), + default_value=0.2, + log=False), + translate_percent_offset: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="translate_percent_offset", + value_range=(0, 0.4), + default_value=0.2, + log=False), + shear: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="shear", + value_range=(0, 45), + default_value=30, + log=False), + rotate: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="rotate", + value_range=(0, 360), + default_value=45, + log=False), ) -> ConfigurationSpace: - cs = ConfigurationSpace() - scale_offset = UniformFloatHyperparameter('scale_offset', lower=scale_offset[0][0], upper=scale_offset[0][1], - default_value=scale_offset[1]) - - translate_percent_offset = UniformFloatHyperparameter('translate_percent_offset', - lower=translate_percent_offset[0][0], - upper=translate_percent_offset[0][1], - default_value=translate_percent_offset[1]) - shear = UniformIntegerHyperparameter('shear', lower=shear[0][0], upper=shear[0][1], default_value=shear[1]) - rotate = UniformIntegerHyperparameter('rotate', lower=0, upper=360, default_value=45) - - use_augmenter = CategoricalHyperparameter('use_augmenter', choices=use_augmenter[0], - default_value=use_augmenter[1]) - cs.add_hyperparameters([scale_offset, translate_percent_offset]) - cs.add_hyperparameters([shear, rotate, use_augmenter]) + use_augmenter = get_hyperparameter(use_augmenter, CategoricalHyperparameter) + scale_offset = get_hyperparameter(scale_offset, UniformFloatHyperparameter) + translate_percent_offset = get_hyperparameter(translate_percent_offset, UniformFloatHyperparameter) + shear = get_hyperparameter(shear, UniformIntegerHyperparameter) + rotate = get_hyperparameter(rotate, UniformIntegerHyperparameter) + cs.add_hyperparameters([use_augmenter, scale_offset, translate_percent_offset]) + cs.add_hyperparameters([shear, rotate]) # only add hyperparameters to configuration space if we are using the augmenter cs.add_condition(CS.EqualsCondition(scale_offset, use_augmenter, True)) diff --git a/autoPyTorch/pipeline/components/setup/augmentation/image/RandomCutout.py b/autoPyTorch/pipeline/components/setup/augmentation/image/RandomCutout.py index 332efbb41..ca8c1f15e 100644 --- a/autoPyTorch/pipeline/components/setup/augmentation/image/RandomCutout.py +++ b/autoPyTorch/pipeline/components/setup/augmentation/image/RandomCutout.py @@ -13,6 +13,7 @@ import numpy as np from autoPyTorch.pipeline.components.setup.augmentation.image.base_image_augmenter import BaseImageAugmenter +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter class RandomCutout(BaseImageAugmenter): @@ -31,16 +32,21 @@ def fit(self, X: Dict[str, Any], y: Any = None) -> BaseImageAugmenter: @staticmethod def get_hyperparameter_search_space( - dataset_properties: Optional[Dict[str, str]] = None, - use_augmenter: Tuple[Tuple, bool] = ((True, False), True), - p: Tuple[Tuple, float] = ((0.2, 1.0), 0.5) + dataset_properties: Optional[Dict[str, str]] = None, + use_augmenter: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_augmenter", + value_range=(True, False), + default_value=True, + log=False), + p: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="p", + value_range=(0.2, 1.0), + default_value=0.5, + log=False), ) -> ConfigurationSpace: cs = ConfigurationSpace() - p = UniformFloatHyperparameter('p', lower=p[0][0], upper=p[0][1], default_value=p[1]) - use_augmenter = CategoricalHyperparameter('use_augmenter', choices=use_augmenter[0], - default_value=use_augmenter[1]) - cs.add_hyperparameters([p, use_augmenter]) + use_augmenter = get_hyperparameter(use_augmenter, CategoricalHyperparameter) + p = get_hyperparameter(p, UniformFloatHyperparameter) + cs.add_hyperparameters([use_augmenter, p]) # only add hyperparameters to configuration space if we are using the augmenter cs.add_condition(CS.EqualsCondition(p, use_augmenter, True)) return cs diff --git a/autoPyTorch/pipeline/components/setup/augmentation/image/Resize.py b/autoPyTorch/pipeline/components/setup/augmentation/image/Resize.py index eabafa95b..d05ab970f 100644 --- a/autoPyTorch/pipeline/components/setup/augmentation/image/Resize.py +++ b/autoPyTorch/pipeline/components/setup/augmentation/image/Resize.py @@ -11,7 +11,7 @@ import numpy as np from autoPyTorch.pipeline.components.setup.augmentation.image.base_image_augmenter import BaseImageAugmenter -from autoPyTorch.utils.common import FitRequirement +from autoPyTorch.utils.common import FitRequirement, HyperparameterSearchSpace, add_hyperparameter class Resize(BaseImageAugmenter): @@ -35,13 +35,14 @@ def fit(self, X: Dict[str, Any], y: Any = None) -> BaseImageAugmenter: @staticmethod def get_hyperparameter_search_space( - dataset_properties: Optional[Dict[str, str]] = None, - use_augmenter: Tuple[Tuple, bool] = ((True, False), True), + dataset_properties: Optional[Dict[str, str]] = None, + use_augmenter: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_augmenter", + value_range=(True, False), + default_value=True, + log=False), ) -> ConfigurationSpace: cs = ConfigurationSpace() - use_augmenter = CategoricalHyperparameter('use_augmenter', choices=use_augmenter[0], - default_value=use_augmenter[1]) - cs.add_hyperparameters([use_augmenter]) + add_hyperparameter(cs, use_augmenter, CategoricalHyperparameter) return cs diff --git a/autoPyTorch/pipeline/components/setup/augmentation/image/ZeroPadAndCrop.py b/autoPyTorch/pipeline/components/setup/augmentation/image/ZeroPadAndCrop.py index 257c0a550..a3ceb7480 100644 --- a/autoPyTorch/pipeline/components/setup/augmentation/image/ZeroPadAndCrop.py +++ b/autoPyTorch/pipeline/components/setup/augmentation/image/ZeroPadAndCrop.py @@ -11,7 +11,7 @@ import numpy as np from autoPyTorch.pipeline.components.setup.augmentation.image.base_image_augmenter import BaseImageAugmenter -from autoPyTorch.utils.common import FitRequirement +from autoPyTorch.utils.common import FitRequirement, HyperparameterSearchSpace, add_hyperparameter class ZeroPadAndCrop(BaseImageAugmenter): @@ -42,13 +42,14 @@ def fit(self, X: Dict[str, Any], y: Any = None) -> BaseImageAugmenter: @staticmethod def get_hyperparameter_search_space( dataset_properties: Optional[Dict[str, str]] = None, - percent: Tuple[Tuple, float] = ((0, 0.5), 0.1) + percent: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='percent', + value_range=(0, 0.5), + default_value=0.1, + log=False) ) -> ConfigurationSpace: cs = ConfigurationSpace() - percent = UniformFloatHyperparameter('percent', lower=percent[0][0], upper=percent[0][1], - default_value=percent[1]) - cs.add_hyperparameters([percent]) + add_hyperparameter(cs, percent, UniformFloatHyperparameter) return cs @staticmethod diff --git a/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingLR.py b/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingLR.py index 9cbbaa41d..8c030e78e 100644 --- a/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingLR.py +++ b/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingLR.py @@ -11,6 +11,7 @@ from torch.optim.lr_scheduler import _LRScheduler from autoPyTorch.pipeline.components.setup.lr_scheduler.base_scheduler import BaseLRComponent +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter class CosineAnnealingLR(BaseLRComponent): @@ -56,16 +57,18 @@ def fit(self, X: Dict[str, Any], y: Any = None) -> BaseLRComponent: @staticmethod def get_properties(dataset_properties: Optional[Dict[str, Any]] = None) -> Dict[str, str]: return { - 'shortname': 'CosineAnnealingWarmRestarts', - 'name': 'Cosine Annealing WarmRestarts', + 'shortname': 'CosineAnnealing', + 'name': 'Cosine Annealing', } @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, - T_max: Tuple[Tuple[int, int], int] = ((10, 500), 200) - ) -> ConfigurationSpace: - T_max = UniformIntegerHyperparameter( - "T_max", T_max[0][0], T_max[0][1], default_value=T_max[1]) + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict] = None, + T_max: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='T_max', + value_range=(10, 500), + default_value=200, + log=False) + ) -> ConfigurationSpace: cs = ConfigurationSpace() - cs.add_hyperparameters([T_max]) + add_hyperparameter(cs, T_max, UniformIntegerHyperparameter) return cs diff --git a/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingWarmRestarts.py b/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingWarmRestarts.py index 07c9ffc2a..bf94a4e83 100644 --- a/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingWarmRestarts.py +++ b/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingWarmRestarts.py @@ -9,6 +9,7 @@ from torch.optim.lr_scheduler import _LRScheduler from autoPyTorch.pipeline.components.setup.lr_scheduler.base_scheduler import BaseLRComponent +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter class CosineAnnealingWarmRestarts(BaseLRComponent): @@ -30,7 +31,6 @@ def __init__( T_mult: int, random_state: Optional[np.random.RandomState] = None ): - super().__init__() self.T_0 = T_0 self.T_mult = T_mult @@ -67,14 +67,19 @@ def get_properties(dataset_properties: Optional[Dict[str, Any]] = None) -> Dict[ } @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, - T_0: Tuple[Tuple[int, int], int] = ((1, 20), 1), - T_mult: Tuple[Tuple[float, float], float] = ((1.0, 2.0), 1.0) - ) -> ConfigurationSpace: - T_0 = UniformIntegerHyperparameter( - "T_0", T_0[0][0], T_0[0][1], default_value=T_0[1]) - T_mult = UniformFloatHyperparameter( - "T_mult", T_mult[0][0], T_mult[0][1], default_value=T_mult[1]) + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict] = None, + T_0: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='T_0', + value_range=(1, 20), + default_value=1, + log=False), + T_mult: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='T_mult', + value_range=(1.0, 2.0), + default_value=1.0, + log=False), + ) -> ConfigurationSpace: cs = ConfigurationSpace() - cs.add_hyperparameters([T_0, T_mult]) + add_hyperparameter(cs, T_0, UniformIntegerHyperparameter) + add_hyperparameter(cs, T_mult, UniformFloatHyperparameter) + return cs diff --git a/autoPyTorch/pipeline/components/setup/lr_scheduler/CyclicLR.py b/autoPyTorch/pipeline/components/setup/lr_scheduler/CyclicLR.py index bc6e4e3ff..8d0fe9c37 100644 --- a/autoPyTorch/pipeline/components/setup/lr_scheduler/CyclicLR.py +++ b/autoPyTorch/pipeline/components/setup/lr_scheduler/CyclicLR.py @@ -13,6 +13,7 @@ from torch.optim.lr_scheduler import _LRScheduler from autoPyTorch.pipeline.components.setup.lr_scheduler.base_scheduler import BaseLRComponent +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter class CyclicLR(BaseLRComponent): @@ -31,6 +32,7 @@ class CyclicLR(BaseLRComponent): base_lr. This simplifies the learning space """ + def __init__( self, base_lr: float, @@ -39,7 +41,6 @@ def __init__( max_lr: float = 0.1, random_state: Optional[np.random.RandomState] = None ): - super().__init__() self.base_lr = base_lr self.mode = mode @@ -82,24 +83,36 @@ def fit(self, X: Dict[str, Any], y: Any = None) -> BaseLRComponent: def get_properties(dataset_properties: Optional[Dict[str, Any]] = None) -> Dict[str, str]: return { 'shortname': 'CyclicLR', - 'name': 'CyclicLR', + 'name': 'Cyclic Learning Rate Scheduler', } @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, - base_lr: Tuple[Tuple, float] = ((1e-6, 1e-1), 0.01), - mode: Tuple[Tuple, str] = (('triangular', 'triangular2', 'exp_range'), - 'triangular'), - step_size_up: Tuple[Tuple, int] = ((1000, 4000), 2000), - max_lr: Tuple[Tuple, float] = ((1e-3, 1e-1), 0.1) - ) -> ConfigurationSpace: - base_lr = UniformFloatHyperparameter( - "base_lr", base_lr[0][0], base_lr[0][1], default_value=base_lr[1]) - mode = CategoricalHyperparameter('mode', choices=mode[0], default_value=mode[1]) - step_size_up = UniformIntegerHyperparameter( - "step_size_up", step_size_up[0][0], step_size_up[0][1], default_value=step_size_up[1]) - max_lr = UniformFloatHyperparameter( - "max_lr", max_lr[0][0], max_lr[0][1], default_value=max_lr[1]) + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict] = None, + base_lr: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='base_lr', + value_range=(1e-6, 1e-1), + default_value=0.01, + log=False), + mode: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='mode', + value_range=('triangular', + 'triangular2', + 'exp_range'), + default_value='triangular', + log=False), + step_size_up: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='step_size_up', + value_range=(1000, 4000), + default_value=2000, + log=False), + max_lr: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='max_lr', + value_range=(1e-3, 1e-1), + default_value=0.1, + log=False) + ) -> ConfigurationSpace: cs = ConfigurationSpace() - cs.add_hyperparameters([base_lr, mode, step_size_up, max_lr]) + + add_hyperparameter(cs, base_lr, UniformFloatHyperparameter) + add_hyperparameter(cs, mode, CategoricalHyperparameter) + add_hyperparameter(cs, step_size_up, UniformIntegerHyperparameter) + add_hyperparameter(cs, max_lr, UniformFloatHyperparameter) + return cs diff --git a/autoPyTorch/pipeline/components/setup/lr_scheduler/ExponentialLR.py b/autoPyTorch/pipeline/components/setup/lr_scheduler/ExponentialLR.py index d090d710b..e7de57679 100644 --- a/autoPyTorch/pipeline/components/setup/lr_scheduler/ExponentialLR.py +++ b/autoPyTorch/pipeline/components/setup/lr_scheduler/ExponentialLR.py @@ -11,6 +11,7 @@ from torch.optim.lr_scheduler import _LRScheduler from autoPyTorch.pipeline.components.setup.lr_scheduler.base_scheduler import BaseLRComponent +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter class ExponentialLR(BaseLRComponent): @@ -58,15 +59,17 @@ def fit(self, X: Dict[str, Any], y: Any = None) -> BaseLRComponent: def get_properties(dataset_properties: Optional[Dict[str, Any]] = None) -> Dict[str, str]: return { 'shortname': 'ExponentialLR', - 'name': 'ExponentialLR', + 'name': 'Exponential Learning Rate Scheduler', } @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, - gamma: Tuple[Tuple, float] = ((0.7, 0.9999), 0.9) - ) -> ConfigurationSpace: - gamma = UniformFloatHyperparameter( - "gamma", gamma[0][0], gamma[0][1], default_value=gamma[1]) + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict] = None, + gamma: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='gamma', + value_range=(0.7, 0.9999), + default_value=0.9, + log=False) + ) -> ConfigurationSpace: cs = ConfigurationSpace() - cs.add_hyperparameters([gamma]) + add_hyperparameter(cs, gamma, UniformFloatHyperparameter) return cs diff --git a/autoPyTorch/pipeline/components/setup/lr_scheduler/ReduceLROnPlateau.py b/autoPyTorch/pipeline/components/setup/lr_scheduler/ReduceLROnPlateau.py index bd8c9c97a..48ab46722 100644 --- a/autoPyTorch/pipeline/components/setup/lr_scheduler/ReduceLROnPlateau.py +++ b/autoPyTorch/pipeline/components/setup/lr_scheduler/ReduceLROnPlateau.py @@ -13,6 +13,7 @@ from torch.optim.lr_scheduler import _LRScheduler from autoPyTorch.pipeline.components.setup.lr_scheduler.base_scheduler import BaseLRComponent +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter class ReduceLROnPlateau(BaseLRComponent): @@ -31,6 +32,7 @@ class ReduceLROnPlateau(BaseLRComponent): rate will be reduced. random_state (Optional[np.random.RandomState]): random state """ + def __init__( self, mode: str, @@ -38,7 +40,6 @@ def __init__( patience: int, random_state: Optional[np.random.RandomState] = None ): - super().__init__() self.mode = mode self.factor = factor @@ -77,16 +78,26 @@ def get_properties(dataset_properties: Optional[Dict[str, Any]] = None) -> Dict[ } @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, - mode: Tuple[Tuple, str] = (('min', 'max'), 'min'), - patience: Tuple[Tuple, int] = ((5, 20), 10), - factor: Tuple[Tuple[float, float], float] = ((0.01, 0.9), 0.1) - ) -> ConfigurationSpace: - mode = CategoricalHyperparameter('mode', choices=mode[0], default_value=mode[1]) - patience = UniformIntegerHyperparameter( - "patience", patience[0][0], patience[0][1], default_value=patience[1]) - factor = UniformFloatHyperparameter( - "factor", factor[0][0], factor[0][1], default_value=factor[1]) + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict] = None, + mode: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='mode', + value_range=('min', 'max'), + default_value='min', + log=False), + patience: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='patience', + value_range=(5, 20), + default_value=10, + log=False), + factor: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='factor', + value_range=(0.01, 0.9), + default_value=0.1, + log=False), + ) -> ConfigurationSpace: + cs = ConfigurationSpace() - cs.add_hyperparameters([mode, patience, factor]) + + add_hyperparameter(cs, mode, CategoricalHyperparameter) + add_hyperparameter(cs, patience, UniformIntegerHyperparameter) + add_hyperparameter(cs, factor, UniformFloatHyperparameter) + return cs diff --git a/autoPyTorch/pipeline/components/setup/lr_scheduler/StepLR.py b/autoPyTorch/pipeline/components/setup/lr_scheduler/StepLR.py index 035e4e841..c504db198 100644 --- a/autoPyTorch/pipeline/components/setup/lr_scheduler/StepLR.py +++ b/autoPyTorch/pipeline/components/setup/lr_scheduler/StepLR.py @@ -12,6 +12,7 @@ from torch.optim.lr_scheduler import _LRScheduler from autoPyTorch.pipeline.components.setup.lr_scheduler.base_scheduler import BaseLRComponent +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter class StepLR(BaseLRComponent): @@ -68,14 +69,21 @@ def get_properties(dataset_properties: Optional[Dict[str, Any]] = None) -> Dict[ } @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, - gamma: Tuple[Tuple, float] = ((0.001, 0.9), 0.1), - step_size: Tuple[Tuple, int] = ((1, 10), 5) - ) -> ConfigurationSpace: - gamma = UniformFloatHyperparameter( - "gamma", gamma[0][0], gamma[0][1], default_value=gamma[1]) - step_size = UniformIntegerHyperparameter( - "step_size", step_size[0][0], step_size[0][1], default_value=step_size[1]) + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict] = None, + gamma: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='gamma', + value_range=(0.001, 0.9), + default_value=0.1, + log=False), + step_size: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='step_size', + value_range=(1, 10), + default_value=5, + log=False), + ) -> ConfigurationSpace: + cs = ConfigurationSpace() - cs.add_hyperparameters([gamma, step_size]) + + add_hyperparameter(cs, step_size, UniformIntegerHyperparameter) + add_hyperparameter(cs, gamma, UniformFloatHyperparameter) + return cs diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/ConvNetImageBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/ConvNetImageBackbone.py index a9d1855c8..3d5590552 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/ConvNetImageBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/ConvNetImageBackbone.py @@ -11,6 +11,7 @@ from autoPyTorch.pipeline.components.setup.network_backbone.base_network_backbone import NetworkBackboneComponent from autoPyTorch.pipeline.components.setup.network_backbone.utils import _activations +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter class ConvNetImageBackbone(NetworkBackboneComponent): @@ -67,56 +68,45 @@ def get_properties(dataset_properties: Optional[Dict[str, str]] = None) -> Dict[ } @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, - num_layers: Tuple[Tuple, int] = ((2, 8), 4), - num_init_filters: Tuple[Tuple, int] = ((16, 64), 32), - activation: Tuple[Tuple, str] = (tuple(_activations.keys()), - list(_activations.keys())[0]), - kernel_size: Tuple[Tuple, int] = ((3, 5), 3), - stride: Tuple[Tuple, int] = ((1, 3), 1), - padding: Tuple[Tuple, int] = ((2, 3), 2), - pool_size: Tuple[Tuple, int] = ((2, 3), 2) - ) -> ConfigurationSpace: + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict] = None, + num_layers: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='num_layers', + value_range=(2, 8), + default_value=4, + log=False), + num_init_filters: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='num_init_filters', + value_range=(16, 64), + default_value=32, + log=False), + activation: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='activation', + value_range=tuple(_activations.keys()), + default_value=list(_activations.keys())[0], + log=False), + kernel_size: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='kernel_size', + value_range=(3, 5), + default_value=3, + log=False), + stride: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='stride', + value_range=(1, 3), + default_value=1, + log=False), + padding: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='padding', + value_range=(2, 3), + default_value=2, + log=False), + pool_size: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='pool_size', + value_range=(2, 3), + default_value=2, + log=False) + ) -> ConfigurationSpace: cs = CS.ConfigurationSpace() - min_num_layers, max_num_layers = num_layers[0] - cs.add_hyperparameter(UniformIntegerHyperparameter('num_layers', - lower=min_num_layers, - upper=max_num_layers, - default_value=num_layers[1])) - - cs.add_hyperparameter(CategoricalHyperparameter('activation', - choices=activation[0], - default_value=activation[1])) - - min_init_filters, max_init_filters = num_init_filters[0] - cs.add_hyperparameter(UniformIntegerHyperparameter('conv_init_filters', - lower=min_init_filters, - upper=max_init_filters, - default_value=num_init_filters[1])) - - min_kernel_size, max_kernel_size = kernel_size[0] - cs.add_hyperparameter(UniformIntegerHyperparameter('conv_kernel_size', - lower=min_kernel_size, - upper=max_kernel_size, - default_value=kernel_size[1])) - - min_stride, max_stride = stride[0] - cs.add_hyperparameter(UniformIntegerHyperparameter('conv_kernel_stride', - lower=min_stride, - upper=max_stride, - default_value=stride[1])) - - min_padding, max_padding = padding[0] - cs.add_hyperparameter(UniformIntegerHyperparameter('conv_kernel_padding', - lower=min_padding, - upper=max_padding, - default_value=padding[1])) - - min_pool_size, max_pool_size = pool_size[0] - cs.add_hyperparameter(UniformIntegerHyperparameter('pool_size', - lower=min_pool_size, - upper=max_pool_size, - default_value=pool_size[1])) + add_hyperparameter(cs, num_init_filters, UniformIntegerHyperparameter) + add_hyperparameter(cs, num_init_filters, UniformIntegerHyperparameter) + add_hyperparameter(cs, activation, CategoricalHyperparameter) + add_hyperparameter(cs, kernel_size, UniformIntegerHyperparameter) + add_hyperparameter(cs, stride, UniformIntegerHyperparameter) + add_hyperparameter(cs, padding, UniformIntegerHyperparameter) + add_hyperparameter(cs, pool_size, UniformIntegerHyperparameter) return cs diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/DenseNetImageBackone.py b/autoPyTorch/pipeline/components/setup/network_backbone/DenseNetImageBackone.py index 98e0eb9b8..7dd222328 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/DenseNetImageBackone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/DenseNetImageBackone.py @@ -16,6 +16,7 @@ from autoPyTorch.pipeline.components.setup.network_backbone.base_network_backbone import NetworkBackboneComponent from autoPyTorch.pipeline.components.setup.network_backbone.utils import _activations +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter class _DenseLayer(nn.Sequential): @@ -152,51 +153,54 @@ def get_properties(dataset_properties: Optional[Dict[str, str]] = None) -> Dict[ } @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, - num_blocks: Tuple[Tuple, int] = ((3, 4), 3), - num_layers: Tuple[Tuple, int] = ((4, 64), 16), - growth_rate: Tuple[Tuple, int] = ((12, 40), 20), - activation: Tuple[Tuple, str] = (tuple(_activations.keys()), - list(_activations.keys())[0]), - use_dropout: Tuple[Tuple, bool] = ((True, False), False), - dropout: Tuple[Tuple, float] = ((0, 0.5), 0.2) - ) -> ConfigurationSpace: + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict] = None, + num_layers: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='num_layers', + value_range=(4, 64), + default_value=16, + log=False), + num_blocks: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='num_blocks', + value_range=(3, 4), + default_value=3, + log=False), + growth_rate: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='growth_rate', + value_range=(12, 40), + default_value=20, + log=False), + activation: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='activation', + value_range=tuple(_activations.keys()), + default_value=list(_activations.keys())[0], + log=False), + use_dropout: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='use_dropout', + value_range=(True, False), + default_value=False, + log=False), + dropout: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='dropout', + value_range=(0, 0.5), + default_value=0.2, + log=False), + ) -> ConfigurationSpace: cs = CS.ConfigurationSpace() - min_growth_rate, max_growth_rate = growth_rate[0] - growth_rate_hp = UniformIntegerHyperparameter('growth_rate', - lower=min_growth_rate, - upper=max_growth_rate, - default_value=growth_rate[1]) - cs.add_hyperparameter(growth_rate_hp) - - min_num_blocks, max_num_blocks = num_blocks[0] - blocks_hp = UniformIntegerHyperparameter('blocks', - lower=min_num_blocks, - upper=max_num_blocks, - default_value=num_blocks[1]) + min_num_layers, max_num_layers = num_layers.value_range + add_hyperparameter(cs, num_layers, UniformIntegerHyperparameter) + add_hyperparameter(cs, growth_rate, UniformIntegerHyperparameter) + + min_num_blocks, max_num_blocks = num_blocks.value_range + blocks_hp = get_hyperparameter(num_blocks, UniformIntegerHyperparameter) cs.add_hyperparameter(blocks_hp) - activation_hp = CategoricalHyperparameter('activation', - choices=activation[0], - default_value=activation[1]) - cs.add_hyperparameter(activation_hp) + add_hyperparameter(cs, activation, CategoricalHyperparameter) - use_dropout = CategoricalHyperparameter('use_dropout', - choices=use_dropout[0], - default_value=use_dropout[1]) + use_dropout = get_hyperparameter(use_dropout, CategoricalHyperparameter) - min_dropout, max_dropout = dropout[0] - dropout = UniformFloatHyperparameter('dropout', - lower=min_dropout, - upper=max_dropout, - default_value=dropout[1]) + dropout = get_hyperparameter(dropout, UniformFloatHyperparameter) cs.add_hyperparameters([use_dropout, dropout]) cs.add_condition(CS.EqualsCondition(dropout, use_dropout, True)) for i in range(1, max_num_blocks + 1): - min_num_layers, max_num_layers = num_layers[0] + layer_hp = UniformIntegerHyperparameter('layer_in_block_%d' % i, lower=min_num_layers, upper=max_num_layers, diff --git a/autoPyTorch/utils/common.py b/autoPyTorch/utils/common.py index 1fa1c0f8f..dbccedac2 100644 --- a/autoPyTorch/utils/common.py +++ b/autoPyTorch/utils/common.py @@ -1,5 +1,14 @@ import hashlib -from typing import Any, Dict, Iterable, List, NamedTuple, Optional, Type, Union +from typing import Any, Dict, Iterable, List, NamedTuple, Optional, Sequence, Type, Union + +from ConfigSpace.configuration_space import ConfigurationSpace +from ConfigSpace.hyperparameters import ( + Hyperparameter, + CategoricalHyperparameter, + Constant, + UniformIntegerHyperparameter, + UniformFloatHyperparameter +) import numpy as np @@ -10,16 +19,18 @@ import torch from torch.utils.data.dataloader import default_collate +HyperparameterValueType = Union[int, str, float] + class FitRequirement(NamedTuple): """ - A class that holds inputs required to fit a pipeline. Also indicates wether + A class that holds inputs required to fit a pipeline. Also indicates whether requirements have to be user specified or are generated by the pipeline itself. Attributes: - name: The name of the variable expected in the input dictionary - supported_types: An iterable of all types that are supported - user_defined: If false, this requirement does not have to be given to the pipeline + name: The name of the variable expected in the input dictionary + supported_types: An iterable of all types that are supported + user_defined: If false, this requirement does not have to be given to the pipeline """ name: str @@ -35,6 +46,25 @@ def __str__(self) -> str: self.name, self.supported_types, self.user_defined, self.dataset_property) +class HyperparameterSearchSpace(NamedTuple): + """ + A class that holds the search space for an individual hyperparameter. + Attributes: + hyperparameter (str): + name of the hyperparameter + value_range (Iterable[HyperparameterValueType]): + range of the hyperparameter + default_value (HyperparameterValueType): + default value of the hyperparameter + log (bool): + whether to sample hyperparameter on a log scale + """ + hyperparameter: str + value_range: Sequence[HyperparameterValueType] + default_value: HyperparameterValueType + log: bool + + def replace_prefix_in_config_dict(config: Dict[str, Any], prefix: str, replace: str = "") -> Dict[str, Any]: """ Replace the prefix in all keys with the specified replacement string (the empty string by @@ -156,3 +186,68 @@ def subsampler(data: Union[np.ndarray, pd.DataFrame, scipy.sparse.csr_matrix], x: Union[np.ndarray, List[int]] ) -> Union[np.ndarray, pd.DataFrame, scipy.sparse.csr_matrix]: return data[x] if isinstance(data, (np.ndarray, scipy.sparse.csr_matrix)) else data.iloc[x] + + +def get_hyperparameter(hyperparameter: HyperparameterSearchSpace, + hyperparameter_type: Type[Hyperparameter]) -> Hyperparameter: + """ + Given a hyperparameter search space, return a ConfigSpace Hyperparameter + Args: + hyperparameter (HyperparameterSearchSpace): + the search space for the hyperparameter + hyperparameter_type (Hyperparameter): + the type of the hyperparameter + + Returns: + Hyperparameter + """ + if len(hyperparameter.value_range) == 0: + raise ValueError(hyperparameter.hyperparameter + ': The range has to contain at least one element') + if len(hyperparameter.value_range) == 1 and hyperparameter_type != CategoricalHyperparameter: + return Constant(hyperparameter.hyperparameter, int(hyperparameter.value_range[0]) + if isinstance(hyperparameter.value_range[0], bool) else hyperparameter.value_range[0]) + if len(hyperparameter.value_range) == 2 and hyperparameter.value_range[0] == hyperparameter.value_range[1]: + return Constant(hyperparameter.hyperparameter, int(hyperparameter.value_range[0]) + if isinstance(hyperparameter.value_range[0], bool) else hyperparameter.value_range[0]) + if hyperparameter_type == CategoricalHyperparameter: + return CategoricalHyperparameter(hyperparameter.hyperparameter, + choices=hyperparameter.value_range, + default_value=hyperparameter.default_value) + if hyperparameter_type == UniformFloatHyperparameter: + assert len(hyperparameter.value_range) == 2, \ + "Float HP range update for %s is specified by the two upper " \ + "and lower values. %s given." % (hyperparameter.hyperparameter, len(hyperparameter.value_range)) + return UniformFloatHyperparameter(hyperparameter.hyperparameter, + lower=hyperparameter.value_range[0], + upper=hyperparameter.value_range[1], + log=hyperparameter.log, + defaul_value=hyperparameter.default_value) + if hyperparameter_type == UniformIntegerHyperparameter: + assert len(hyperparameter.value_range) == 2, \ + "Int HP range update for %s is specified by the two upper " \ + "and lower values. %s given." % (hyperparameter.hyperparameter, len(hyperparameter.value_range)) + return UniformIntegerHyperparameter(hyperparameter.hyperparameter, + lower=hyperparameter.value_range[0], + upper=hyperparameter.value_range[1], + log=hyperparameter.log, + defaul_value=hyperparameter.default_value) + raise ValueError('Unknown type: %s for hp %s' % (hyperparameter_type, hyperparameter.hyperparameter)) + + +def add_hyperparameter(cs: ConfigurationSpace, + hyperparameter: HyperparameterSearchSpace, + hyperparameter_type: Type[Hyperparameter]): + """ + Adds the given hyperparameter to the given configuration space + Args: + cs (ConfigurationSpace): + Configuration space where the hyperparameter must be added + hyperparameter (HyperparameterSearchSpace): + search space of the hyperparameter + hyperparameter_type (Hyperparameter): + type of the hyperparameter + + Returns: + + """ + return cs.add_hyperparameter(get_hyperparameter(hyperparameter, hyperparameter_type)) diff --git a/autoPyTorch/utils/hyperparameter_search_space_update.py b/autoPyTorch/utils/hyperparameter_search_space_update.py index e2ef1c85f..0ecd22ebe 100644 --- a/autoPyTorch/utils/hyperparameter_search_space_update.py +++ b/autoPyTorch/utils/hyperparameter_search_space_update.py @@ -1,9 +1,10 @@ import ast import os -from typing import List, Optional, Tuple, Union +from typing import List, Optional, Sequence, Tuple, Union from autoPyTorch.pipeline.components.base_choice import autoPyTorchChoice from autoPyTorch.pipeline.components.base_component import autoPyTorchComponent +from autoPyTorch.utils.common import HyperparameterSearchSpace, HyperparameterValueType class HyperparameterSearchSpaceUpdate: @@ -16,19 +17,21 @@ class HyperparameterSearchSpaceUpdate: The name of the node in the pipeline hyperparameter (str): The name of the hyperparameter - value_range (Union[List, Tuple]): + value_range (Sequence[HyperparameterValueType]): In case of categorical hyperparameter, defines the new categorical choices. In case of numerical hyperparameter, defines the new range in the form of (LOWER, UPPER) - default_value (Union[int, float, str]): + default_value (HyperparameterValueType): New default value for the hyperparameter log (bool) (default=False): In case of numerical hyperparameters, whether to sample on a log scale """ - def __init__(self, node_name: str, hyperparameter: str, value_range: Union[List, Tuple], - default_value: Union[int, float, str], log: bool = False) -> None: + def __init__(self, node_name: str, hyperparameter: str, value_range: Sequence[HyperparameterValueType], + default_value: HyperparameterValueType, log: bool = False) -> None: self.node_name = node_name self.hyperparameter = hyperparameter + if len(value_range) == 0: + raise ValueError("The new value range needs at least one value") self.value_range = value_range self.log = log self.default_value = default_value @@ -43,11 +46,7 @@ def apply(self, pipeline: List[Tuple[str, Union[autoPyTorchComponent, autoPyTorc Returns: None """ - [node[1]._apply_search_space_update(name=self.hyperparameter, - new_value_range=self.value_range, - log=self.log, - default_value=self.default_value) - for node in pipeline if node[0] == self.node_name] + [node[1]._apply_search_space_update(self) for node in pipeline if node[0] == self.node_name] def __str__(self) -> str: return "{}, {}, {}, {}, {}".format(self.node_name, self.hyperparameter, str(self.value_range), @@ -55,6 +54,12 @@ def __str__(self) -> str: str) else self.default_value, (" log" if self.log else "")) + def get_search_space(self) -> HyperparameterSearchSpace: + return HyperparameterSearchSpace(hyperparameter=self.hyperparameter, + value_range=self.value_range, + default_value=self.default_value, + log=self.log) + class HyperparameterSearchSpaceUpdates: """ Contains a collection of HyperparameterSearchSpaceUpdate """ From ab71b78b8285f8727d5bdad6d135ca6f6d920ff8 Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Mon, 22 Mar 2021 15:25:39 +0100 Subject: [PATCH 02/10] Added SearchSpace working functionality --- autoPyTorch/pipeline/base_pipeline.py | 27 +++- .../pipeline/components/base_choice.py | 12 +- .../pipeline/components/base_component.py | 22 ++- .../feature_preprocessing/KernelPCA.py | 6 +- .../feature_preprocessing/Nystroem.py | 6 +- .../PolynomialFeatures.py | 4 +- .../feature_preprocessing/PowerTransformer.py | 2 +- .../RandomKitchenSinks.py | 2 +- .../feature_preprocessing/TruncatedSVD.py | 4 +- .../base_feature_preprocessor_choice.py | 2 +- .../imputation/SimpleImputer.py | 16 +- .../scaling/Normalizer.py | 2 +- .../setup/augmentation/image/GaussianBlur.py | 6 +- .../setup/augmentation/image/GaussianNoise.py | 4 +- .../setup/augmentation/image/RandomAffine.py | 10 +- .../setup/augmentation/image/RandomCutout.py | 4 +- .../setup/augmentation/image/Resize.py | 2 +- .../augmentation/image/ZeroPadAndCrop.py | 2 +- .../setup/lr_scheduler/CosineAnnealingLR.py | 2 +- .../CosineAnnealingWarmRestarts.py | 4 +- .../components/setup/lr_scheduler/CyclicLR.py | 8 +- .../setup/lr_scheduler/ExponentialLR.py | 2 +- .../setup/lr_scheduler/ReduceLROnPlateau.py | 6 +- .../components/setup/lr_scheduler/StepLR.py | 11 +- .../network_backbone/ConvNetImageBackbone.py | 14 +- .../network_backbone/DenseNetImageBackone.py | 23 ++- .../network_backbone/InceptionTimeBackbone.py | 58 +++---- .../setup/network_backbone/MLPBackbone.py | 71 +++++---- .../setup/network_backbone/ResNetBackbone.py | 142 ++++++++++-------- .../network_backbone/ShapedMLPBackbone.py | 86 +++++------ .../network_backbone/ShapedResNetBackbone.py | 121 ++++++++------- .../setup/network_backbone/TCNBackbone.py | 60 ++++---- .../LearnedEntityEmbedding.py | 30 ++-- .../setup/network_head/fully_connected.py | 43 +++--- .../setup/network_head/fully_convolutional.py | 42 ++++-- .../base_network_initializer.py | 16 +- .../setup/optimizer/AdamOptimizer.py | 42 +++--- .../setup/optimizer/AdamWOptimizer.py | 42 +++--- .../setup/optimizer/RMSpropOptimizer.py | 42 +++--- .../setup/optimizer/SGDOptimizer.py | 32 ++-- .../training/data_loader/base_data_loader.py | 27 ++-- .../training/trainer/MixUpTrainer.py | 23 +-- .../training/trainer/StandardTrainer.py | 15 +- autoPyTorch/utils/common.py | 9 +- .../hyperparameter_search_space_update.py | 32 +++- 45 files changed, 611 insertions(+), 525 deletions(-) diff --git a/autoPyTorch/pipeline/base_pipeline.py b/autoPyTorch/pipeline/base_pipeline.py index 30d63519f..1bc94d3a5 100644 --- a/autoPyTorch/pipeline/base_pipeline.py +++ b/autoPyTorch/pipeline/base_pipeline.py @@ -365,7 +365,7 @@ def _get_base_search_space( cs.add_configuration_space( node_name, node.get_hyperparameter_search_space(dataset_properties, # type: ignore[arg-type] - **node._get_search_space_updates(node_name)), + **node._get_search_space_updates()), ) # If the node is a choice, we have to figure out which of its # choices are actually legal choices @@ -398,24 +398,47 @@ def _check_search_space_updates(self, include: Optional[Dict[str, Any]], raise ValueError("Unknown node name. Expected update node name to be in {} " "got {}".format(self.named_steps.keys(), update.node_name)) node = self.named_steps[update.node_name] + # if node is a choice module if hasattr(node, 'get_components'): split_hyperparameter = update.hyperparameter.split(':') + # check if component is not present in include if include is not None and update.node_name in include.keys(): if split_hyperparameter[0] not in include[update.node_name]: raise ValueError("Not found {} in include".format(split_hyperparameter[0])) + # check if component is present in exclude if exclude is not None and update.node_name in exclude.keys(): if split_hyperparameter[0] in exclude[update.node_name]: raise ValueError("Found {} in exclude".format(split_hyperparameter[0])) components = node.get_components() - if split_hyperparameter[0] not in components.keys(): + # if hyperparameter is __choice__, check if + # the components in the value range of search space update + # are in components of the choice module + if split_hyperparameter[0] == '__choice__': + for component in update.value_range: + if include is not None and update.node_name in include.keys(): + if component not in include[update.node_name]: + raise ValueError("Not found {} in include".format(component)) + if exclude is not None and update.node_name in exclude.keys(): + if component in exclude[update.node_name]: + raise ValueError("Found {} in exclude".format(component)) + if component not in components.keys(): + raise ValueError("Unknown hyperparameter for choice {}. " + "Expected update hyperparameter " + "to be in {} got {}".format(node.__class__.__name__, + components.keys(), component)) + # check if the component whose hyperparameter + # needs to be updated is in components of the + # choice module + elif split_hyperparameter[0] not in components.keys(): raise ValueError("Unknown hyperparameter for choice {}. " "Expected update hyperparameter " "to be in {} got {}".format(node.__class__.__name__, components.keys(), split_hyperparameter[0])) else: + # check if hyperparameter is in the search space of the component component = components[split_hyperparameter[0]] if split_hyperparameter[1] not in component. \ get_hyperparameter_search_space(dataset_properties=self.dataset_properties): diff --git a/autoPyTorch/pipeline/components/base_choice.py b/autoPyTorch/pipeline/components/base_choice.py index d222cc6d2..dabdb7a62 100644 --- a/autoPyTorch/pipeline/components/base_choice.py +++ b/autoPyTorch/pipeline/components/base_choice.py @@ -1,5 +1,6 @@ import warnings from collections import OrderedDict +import re from typing import Any, Dict, List, Optional, Tuple, Union from ConfigSpace.configuration_space import Configuration, ConfigurationSpace @@ -259,7 +260,7 @@ def _apply_search_space_update(self, hyperparameter_search_space_update: Hyperpa self._cs_updates[hyperparameter_search_space_update.hyperparameter] = hyperparameter_search_space_update - def _get_search_space_updates(self, hyperparameter: str) -> Dict[str, HyperparameterSearchSpace]: + def _get_search_space_updates(self, prefix: Optional[str] = None) -> Dict[str, HyperparameterSearchSpace]: """Get the search space updates with the given prefix Keyword Arguments: @@ -268,11 +269,16 @@ def _get_search_space_updates(self, hyperparameter: str) -> Dict[str, Hyperparam Returns: dict -- Mapping of search space updates. Keys don't contain the prefix. """ + RETURN_ALL = False + if prefix is None: + RETURN_ALL = True result: Dict[str, HyperparameterSearchSpace] = dict() # iterate over all search space updates of this node and keep the ones that have the given prefix for key in self._cs_updates.keys(): - if hyperparameter == key: - result[hyperparameter] = self._cs_updates[hyperparameter].get_search_space() + if RETURN_ALL: + result[key] = self._cs_updates[key].get_search_space() + elif re.search(f'^{prefix}', key) is not None: + result[key[len(prefix) + 1:]] = self._cs_updates[key].get_search_space(remove_prefix=prefix) return result diff --git a/autoPyTorch/pipeline/components/base_component.py b/autoPyTorch/pipeline/components/base_component.py index 81845a6f5..f475b9f24 100644 --- a/autoPyTorch/pipeline/components/base_component.py +++ b/autoPyTorch/pipeline/components/base_component.py @@ -1,6 +1,7 @@ import importlib import inspect import pkgutil +import re import sys import warnings from collections import OrderedDict @@ -15,9 +16,9 @@ def find_components( - package: str, - directory: str, - base_class: BaseEstimator + package: str, + directory: str, + base_class: BaseEstimator ) -> Dict[str, BaseEstimator]: """Utility to find component on a given directory, that inherit from base_class @@ -36,7 +37,7 @@ def find_components( for member_name, obj in inspect.getmembers(module): if inspect.isclass(obj) and issubclass(obj, base_class) and \ - obj != base_class: + obj != base_class: # TODO test if the obj implements the interface # Keep in mind that this only instantiates the ensemble_wrapper, # but not the real target classifier @@ -141,7 +142,7 @@ def get_properties(dataset_properties: Optional[Dict[str, str]] = None @staticmethod def get_hyperparameter_search_space( - dataset_properties: Optional[Dict[str, str]] = None + dataset_properties: Optional[Dict[str, str]] = None ) -> ConfigurationSpace: """Return the configuration space of this classification algorithm. @@ -265,7 +266,7 @@ def _apply_search_space_update(self, hyperparameter_search_space_update: Hyperpa self._cs_updates[hyperparameter_search_space_update.hyperparameter] = hyperparameter_search_space_update - def _get_search_space_updates(self, hyperparameter: str) -> Dict[str, HyperparameterSearchSpace]: + def _get_search_space_updates(self, prefix: Optional[str] = None) -> Dict[str, HyperparameterSearchSpace]: """Get the search space updates with the given prefix Keyword Arguments: @@ -274,11 +275,16 @@ def _get_search_space_updates(self, hyperparameter: str) -> Dict[str, Hyperparam Returns: dict -- Mapping of search space updates. Keys don't contain the prefix. """ + RETURN_ALL = False + if prefix is None: + RETURN_ALL = True result: Dict[str, HyperparameterSearchSpace] = dict() # iterate over all search space updates of this node and keep the ones that have the given prefix for key in self._cs_updates.keys(): - if hyperparameter == key: - result[hyperparameter] = self._cs_updates[hyperparameter].get_search_space() + if RETURN_ALL: + result[key] = self._cs_updates[key].get_search_space() + elif re.search(f'^{prefix}', key) is not None: + result[key[len(prefix) + 1:]] = self._cs_updates[key].get_search_space(remove_prefix=prefix) return result diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/KernelPCA.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/KernelPCA.py index b9a370429..8a62ba0ea 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/KernelPCA.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/KernelPCA.py @@ -51,11 +51,11 @@ def get_hyperparameter_search_space( n_components: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='n_components', value_range=(0.5, 0.9), default_value=0.5, - log=False), + ), kernel: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='kernel', value_range=('poly', 'rbf', 'sigmoid', 'cosine'), default_value='rbf', - log=False), + ), gamma: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='gamma', value_range=(3.0517578125e-05, 8), default_value=0.01, @@ -67,7 +67,7 @@ def get_hyperparameter_search_space( coef0: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='coef0', value_range=(-1, 1), default_value=0, - log=False) + ) ) -> ConfigurationSpace: cs = ConfigurationSpace() diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/Nystroem.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/Nystroem.py index d73aac3a9..d0f7a237a 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/Nystroem.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/Nystroem.py @@ -48,11 +48,11 @@ def get_hyperparameter_search_space( n_components: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='n_components', value_range=(0.5, 0.9), default_value=0.5, - log=False), + ), kernel: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='kernel', value_range=('poly', 'rbf', 'sigmoid', 'cosine'), default_value='rbf', - log=False), + ), gamma: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='gamma', value_range=(3.0517578125e-05, 8), default_value=0.01, @@ -64,7 +64,7 @@ def get_hyperparameter_search_space( coef0: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='coef0', value_range=(-1, 1), default_value=0, - log=False) + ) ) -> ConfigurationSpace: cs = ConfigurationSpace() diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/PolynomialFeatures.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/PolynomialFeatures.py index 2a51ecbbf..dc05ed86f 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/PolynomialFeatures.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/PolynomialFeatures.py @@ -49,11 +49,11 @@ def get_hyperparameter_search_space( interaction_only: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='interaction_only', value_range=(True, False), default_value=False, - log=False), + ), include_bias: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='include_bias', value_range=(True, False), default_value=False, - log=False), + ), ) -> ConfigurationSpace: cs = ConfigurationSpace() add_hyperparameter(cs, degree, UniformIntegerHyperparameter) diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/PowerTransformer.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/PowerTransformer.py index 95498d1be..3e7a3cf5d 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/PowerTransformer.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/PowerTransformer.py @@ -41,7 +41,7 @@ def get_hyperparameter_search_space( standardize: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='standardize', value_range=(True, False), default_value=True, - log=False), + ), ) -> ConfigurationSpace: cs = ConfigurationSpace() diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/RandomKitchenSinks.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/RandomKitchenSinks.py index c7f60da4a..acdad0594 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/RandomKitchenSinks.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/RandomKitchenSinks.py @@ -38,7 +38,7 @@ def get_hyperparameter_search_space( n_components: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='n_components', value_range=(0.5, 0.9), default_value=0.5, - log=False), + ), gamma: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='gamma', value_range=(3.0517578125e-05, 8), default_value=0.01, diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/TruncatedSVD.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/TruncatedSVD.py index 41c74d269..5aeea161d 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/TruncatedSVD.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/TruncatedSVD.py @@ -42,7 +42,7 @@ def get_hyperparameter_search_space( target_dim: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='target_dim', value_range=(0.5, 0.9), default_value=0.5, - log=False), + ), ) -> ConfigurationSpace: cs = ConfigurationSpace() @@ -51,7 +51,7 @@ def get_hyperparameter_search_space( target_dim = HyperparameterSearchSpace(hyperparameter=target_dim.hyperparameter, value_range=(floor(target_dim.value_range[0] * n_features), floor(target_dim.value_range[1] * n_features)), - default_value=floor(target_dim[1] * n_features), + default_value=floor(target_dim.default_value * n_features), log=target_dim.log) else: target_dim = HyperparameterSearchSpace(hyperparameter=target_dim.hyperparameter, diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/base_feature_preprocessor_choice.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/base_feature_preprocessor_choice.py index b8b17e38d..56af71877 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/base_feature_preprocessor_choice.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/base_feature_preprocessor_choice.py @@ -104,7 +104,7 @@ def get_hyperparameter_search_space(self, # add only child hyperparameters of early_preprocessor choices for name in preprocessor.choices: - updates = self._get_search_space_updates(hyperparameter=name) + updates = self._get_search_space_updates(prefix=name) config_space = available_[name].get_hyperparameter_search_space(dataset_properties, # type:ignore **updates) parent_hyperparameter = {'parent': preprocessor, 'value': name} diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/imputation/SimpleImputer.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/imputation/SimpleImputer.py index 5b933eac2..4db01dd37 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/imputation/SimpleImputer.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/imputation/SimpleImputer.py @@ -12,6 +12,7 @@ from autoPyTorch.pipeline.components.preprocessing.tabular_preprocessing.imputation.base_imputer import BaseImputer from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter + class SimpleImputer(BaseImputer): """ Impute missing values for categorical columns with '!missing!' @@ -72,13 +73,14 @@ def get_hyperparameter_search_space( "most_frequent", "constant_zero"), default_value="mean", - log=False), - categorical_strategy: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='numerical_strategy', - value_range=("most_frequent", - "constant_!missing!"), - default_value="most_frequent", - log=False) - ) -> ConfigurationSpace: + ), + categorical_strategy: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter='categorical_strategy', + value_range=("most_frequent", + "constant_!missing!"), + default_value="most_frequent", + ) + ) -> ConfigurationSpace: cs = ConfigurationSpace() assert dataset_properties is not None, "To create hyperparameter search space" \ ", dataset_properties should not be None" diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/Normalizer.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/Normalizer.py index acdf6ac64..962904d70 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/Normalizer.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/Normalizer.py @@ -42,7 +42,7 @@ def get_hyperparameter_search_space( norm: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="norm", value_range=("mean_abs", "mean_squared", "max"), default_value="mean_squared", - log=False) + ) ) -> ConfigurationSpace: cs = ConfigurationSpace() add_hyperparameter(cs, norm, CategoricalHyperparameter) diff --git a/autoPyTorch/pipeline/components/setup/augmentation/image/GaussianBlur.py b/autoPyTorch/pipeline/components/setup/augmentation/image/GaussianBlur.py index 8586ff47f..f57f00d2d 100644 --- a/autoPyTorch/pipeline/components/setup/augmentation/image/GaussianBlur.py +++ b/autoPyTorch/pipeline/components/setup/augmentation/image/GaussianBlur.py @@ -35,15 +35,15 @@ def get_hyperparameter_search_space( use_augmenter: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_augmenter", value_range=(True, False), default_value=True, - log=False), + ), sigma_min: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="sigma_min", value_range=(0, 3), default_value=0, - log=False), + ), sigma_offset: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="sigma_min", value_range=(0.0, 3.0), default_value=0.5, - log=False), + ), ) -> ConfigurationSpace: cs = ConfigurationSpace() use_augmenter = get_hyperparameter(use_augmenter, CategoricalHyperparameter) diff --git a/autoPyTorch/pipeline/components/setup/augmentation/image/GaussianNoise.py b/autoPyTorch/pipeline/components/setup/augmentation/image/GaussianNoise.py index e3fd399f3..c88dc3136 100644 --- a/autoPyTorch/pipeline/components/setup/augmentation/image/GaussianNoise.py +++ b/autoPyTorch/pipeline/components/setup/augmentation/image/GaussianNoise.py @@ -34,11 +34,11 @@ def get_hyperparameter_search_space( use_augmenter: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_augmenter", value_range=(True, False), default_value=True, - log=False), + ), sigma_offset: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="sigma_offset", value_range=(0.0, 3.0), default_value=0.3, - log=False), + ), ) -> ConfigurationSpace: cs = ConfigurationSpace() diff --git a/autoPyTorch/pipeline/components/setup/augmentation/image/RandomAffine.py b/autoPyTorch/pipeline/components/setup/augmentation/image/RandomAffine.py index 4fe5757ad..fb19bfbca 100644 --- a/autoPyTorch/pipeline/components/setup/augmentation/image/RandomAffine.py +++ b/autoPyTorch/pipeline/components/setup/augmentation/image/RandomAffine.py @@ -42,24 +42,24 @@ def get_hyperparameter_search_space( use_augmenter: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_augmenter", value_range=(True, False), default_value=True, - log=False), + ), scale_offset: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="scale_offset", value_range=(0, 0.4), default_value=0.2, - log=False), + ), translate_percent_offset: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="translate_percent_offset", value_range=(0, 0.4), default_value=0.2, - log=False), + ), shear: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="shear", value_range=(0, 45), default_value=30, - log=False), + ), rotate: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="rotate", value_range=(0, 360), default_value=45, - log=False), + ), ) -> ConfigurationSpace: cs = ConfigurationSpace() diff --git a/autoPyTorch/pipeline/components/setup/augmentation/image/RandomCutout.py b/autoPyTorch/pipeline/components/setup/augmentation/image/RandomCutout.py index ca8c1f15e..b25a594c8 100644 --- a/autoPyTorch/pipeline/components/setup/augmentation/image/RandomCutout.py +++ b/autoPyTorch/pipeline/components/setup/augmentation/image/RandomCutout.py @@ -36,11 +36,11 @@ def get_hyperparameter_search_space( use_augmenter: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_augmenter", value_range=(True, False), default_value=True, - log=False), + ), p: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="p", value_range=(0.2, 1.0), default_value=0.5, - log=False), + ), ) -> ConfigurationSpace: cs = ConfigurationSpace() diff --git a/autoPyTorch/pipeline/components/setup/augmentation/image/Resize.py b/autoPyTorch/pipeline/components/setup/augmentation/image/Resize.py index d05ab970f..f31fae536 100644 --- a/autoPyTorch/pipeline/components/setup/augmentation/image/Resize.py +++ b/autoPyTorch/pipeline/components/setup/augmentation/image/Resize.py @@ -39,7 +39,7 @@ def get_hyperparameter_search_space( use_augmenter: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_augmenter", value_range=(True, False), default_value=True, - log=False), + ), ) -> ConfigurationSpace: cs = ConfigurationSpace() add_hyperparameter(cs, use_augmenter, CategoricalHyperparameter) diff --git a/autoPyTorch/pipeline/components/setup/augmentation/image/ZeroPadAndCrop.py b/autoPyTorch/pipeline/components/setup/augmentation/image/ZeroPadAndCrop.py index a3ceb7480..12f1de85e 100644 --- a/autoPyTorch/pipeline/components/setup/augmentation/image/ZeroPadAndCrop.py +++ b/autoPyTorch/pipeline/components/setup/augmentation/image/ZeroPadAndCrop.py @@ -45,7 +45,7 @@ def get_hyperparameter_search_space( percent: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='percent', value_range=(0, 0.5), default_value=0.1, - log=False) + ) ) -> ConfigurationSpace: cs = ConfigurationSpace() diff --git a/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingLR.py b/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingLR.py index 8c030e78e..d7a5ba727 100644 --- a/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingLR.py +++ b/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingLR.py @@ -67,7 +67,7 @@ def get_hyperparameter_search_space( T_max: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='T_max', value_range=(10, 500), default_value=200, - log=False) + ) ) -> ConfigurationSpace: cs = ConfigurationSpace() add_hyperparameter(cs, T_max, UniformIntegerHyperparameter) diff --git a/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingWarmRestarts.py b/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingWarmRestarts.py index bf94a4e83..72fe63642 100644 --- a/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingWarmRestarts.py +++ b/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingWarmRestarts.py @@ -72,11 +72,11 @@ def get_hyperparameter_search_space( T_0: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='T_0', value_range=(1, 20), default_value=1, - log=False), + ), T_mult: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='T_mult', value_range=(1.0, 2.0), default_value=1.0, - log=False), + ), ) -> ConfigurationSpace: cs = ConfigurationSpace() add_hyperparameter(cs, T_0, UniformIntegerHyperparameter) diff --git a/autoPyTorch/pipeline/components/setup/lr_scheduler/CyclicLR.py b/autoPyTorch/pipeline/components/setup/lr_scheduler/CyclicLR.py index 8d0fe9c37..35e881e0a 100644 --- a/autoPyTorch/pipeline/components/setup/lr_scheduler/CyclicLR.py +++ b/autoPyTorch/pipeline/components/setup/lr_scheduler/CyclicLR.py @@ -92,21 +92,21 @@ def get_hyperparameter_search_space( base_lr: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='base_lr', value_range=(1e-6, 1e-1), default_value=0.01, - log=False), + ), mode: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='mode', value_range=('triangular', 'triangular2', 'exp_range'), default_value='triangular', - log=False), + ), step_size_up: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='step_size_up', value_range=(1000, 4000), default_value=2000, - log=False), + ), max_lr: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='max_lr', value_range=(1e-3, 1e-1), default_value=0.1, - log=False) + ) ) -> ConfigurationSpace: cs = ConfigurationSpace() diff --git a/autoPyTorch/pipeline/components/setup/lr_scheduler/ExponentialLR.py b/autoPyTorch/pipeline/components/setup/lr_scheduler/ExponentialLR.py index e7de57679..489d7ba38 100644 --- a/autoPyTorch/pipeline/components/setup/lr_scheduler/ExponentialLR.py +++ b/autoPyTorch/pipeline/components/setup/lr_scheduler/ExponentialLR.py @@ -68,7 +68,7 @@ def get_hyperparameter_search_space( gamma: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='gamma', value_range=(0.7, 0.9999), default_value=0.9, - log=False) + ) ) -> ConfigurationSpace: cs = ConfigurationSpace() add_hyperparameter(cs, gamma, UniformFloatHyperparameter) diff --git a/autoPyTorch/pipeline/components/setup/lr_scheduler/ReduceLROnPlateau.py b/autoPyTorch/pipeline/components/setup/lr_scheduler/ReduceLROnPlateau.py index 48ab46722..ffb7d6e96 100644 --- a/autoPyTorch/pipeline/components/setup/lr_scheduler/ReduceLROnPlateau.py +++ b/autoPyTorch/pipeline/components/setup/lr_scheduler/ReduceLROnPlateau.py @@ -83,15 +83,15 @@ def get_hyperparameter_search_space( mode: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='mode', value_range=('min', 'max'), default_value='min', - log=False), + ), patience: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='patience', value_range=(5, 20), default_value=10, - log=False), + ), factor: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='factor', value_range=(0.01, 0.9), default_value=0.1, - log=False), + ), ) -> ConfigurationSpace: cs = ConfigurationSpace() diff --git a/autoPyTorch/pipeline/components/setup/lr_scheduler/StepLR.py b/autoPyTorch/pipeline/components/setup/lr_scheduler/StepLR.py index c504db198..eaa578eb1 100644 --- a/autoPyTorch/pipeline/components/setup/lr_scheduler/StepLR.py +++ b/autoPyTorch/pipeline/components/setup/lr_scheduler/StepLR.py @@ -26,13 +26,13 @@ class StepLR(BaseLRComponent): gamma (float) – Multiplicative factor of learning rate decay. Default: 0.1. """ + def __init__( self, step_size: int, gamma: float, random_state: Optional[np.random.RandomState] = None ): - super().__init__() self.gamma = gamma self.step_size = step_size @@ -74,13 +74,12 @@ def get_hyperparameter_search_space( gamma: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='gamma', value_range=(0.001, 0.9), default_value=0.1, - log=False), + ), step_size: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='step_size', - value_range=(1, 10), - default_value=5, - log=False), + value_range=(1, 10), + default_value=5, + ), ) -> ConfigurationSpace: - cs = ConfigurationSpace() add_hyperparameter(cs, step_size, UniformIntegerHyperparameter) diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/ConvNetImageBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/ConvNetImageBackbone.py index 3d5590552..767675216 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/ConvNetImageBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/ConvNetImageBackbone.py @@ -73,31 +73,31 @@ def get_hyperparameter_search_space( num_layers: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='num_layers', value_range=(2, 8), default_value=4, - log=False), + ), num_init_filters: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='num_init_filters', value_range=(16, 64), default_value=32, - log=False), + ), activation: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='activation', value_range=tuple(_activations.keys()), default_value=list(_activations.keys())[0], - log=False), + ), kernel_size: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='kernel_size', value_range=(3, 5), default_value=3, - log=False), + ), stride: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='stride', value_range=(1, 3), default_value=1, - log=False), + ), padding: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='padding', value_range=(2, 3), default_value=2, - log=False), + ), pool_size: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='pool_size', value_range=(2, 3), default_value=2, - log=False) + ) ) -> ConfigurationSpace: cs = CS.ConfigurationSpace() diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/DenseNetImageBackone.py b/autoPyTorch/pipeline/components/setup/network_backbone/DenseNetImageBackone.py index 7dd222328..ea20b1727 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/DenseNetImageBackone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/DenseNetImageBackone.py @@ -158,31 +158,30 @@ def get_hyperparameter_search_space( num_layers: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='num_layers', value_range=(4, 64), default_value=16, - log=False), + ), num_blocks: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='num_blocks', value_range=(3, 4), default_value=3, - log=False), + ), growth_rate: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='growth_rate', value_range=(12, 40), default_value=20, - log=False), + ), activation: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='activation', value_range=tuple(_activations.keys()), default_value=list(_activations.keys())[0], - log=False), + ), use_dropout: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='use_dropout', value_range=(True, False), default_value=False, - log=False), + ), dropout: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='dropout', value_range=(0, 0.5), default_value=0.2, - log=False), + ), ) -> ConfigurationSpace: cs = CS.ConfigurationSpace() - min_num_layers, max_num_layers = num_layers.value_range add_hyperparameter(cs, num_layers, UniformIntegerHyperparameter) add_hyperparameter(cs, growth_rate, UniformIntegerHyperparameter) @@ -201,11 +200,11 @@ def get_hyperparameter_search_space( for i in range(1, max_num_blocks + 1): - layer_hp = UniformIntegerHyperparameter('layer_in_block_%d' % i, - lower=min_num_layers, - upper=max_num_layers, - default_value=num_layers[1]) - cs.add_hyperparameter(layer_hp) + layer_search_space = HyperparameterSearchSpace(hyperparameter='layer_in_block_%d' % i, + value_range=num_layers.value_range, + default_value=num_layers.default_value, + log=num_layers.log) + layer_hp = get_hyperparameter(layer_search_space, UniformIntegerHyperparameter) if i > min_num_blocks: cs.add_condition(CS.GreaterThanCondition(layer_hp, blocks_hp, i - 1)) diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/InceptionTimeBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/InceptionTimeBackbone.py index 4bf5c8842..e9829b300 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/InceptionTimeBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/InceptionTimeBackbone.py @@ -9,6 +9,7 @@ from torch import nn from autoPyTorch.pipeline.components.setup.network_backbone.base_network_backbone import NetworkBackboneComponent +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter # Code inspired by https://github.com/hfawaz/InceptionTime @@ -142,39 +143,30 @@ def get_properties(dataset_properties: Optional[Dict[str, str]] = None) -> Dict[ } @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, - num_blocks: Tuple[Tuple, int] = ((1, 10), 5), - num_filters: Tuple[Tuple, int] = ((4, 64), 32), - kernel_size: Tuple[Tuple, int] = ((4, 64), 32), - bottleneck_size: Tuple[Tuple, int] = ((16, 64), 32) - ) -> ConfigurationSpace: + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict] = None, + num_blocks: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="num_blocks", + value_range=(1, 10), + default_value=5, + ), + num_filters: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="num_filters", + value_range=(4, 64), + default_value=32, + ), + kernel_size: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="kernel_size", + value_range=(4, 64), + default_value=32, + ), + bottleneck_size: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="bottleneck_size", + value_range=(16, 64), + default_value=32, + ), + ) -> ConfigurationSpace: cs = ConfigurationSpace() - min_num_blocks, max_num_blocks = num_blocks[0] - num_blocks_hp = UniformIntegerHyperparameter("num_blocks", - lower=min_num_blocks, - upper=max_num_blocks, - default_value=num_blocks[1]) - cs.add_hyperparameter(num_blocks_hp) - - min_num_filters, max_num_filters = num_filters[0] - num_filters_hp = UniformIntegerHyperparameter("num_filters", - lower=min_num_filters, - upper=max_num_filters, - default_value=num_filters[1]) - cs.add_hyperparameter(num_filters_hp) - - min_bottleneck_size, max_bottleneck_size = bottleneck_size[0] - bottleneck_size_hp = UniformIntegerHyperparameter("bottleneck_size", - lower=min_bottleneck_size, - upper=max_bottleneck_size, - default_value=bottleneck_size[1]) - cs.add_hyperparameter(bottleneck_size_hp) - - min_kernel_size, max_kernel_size = kernel_size[0] - kernel_size_hp = UniformIntegerHyperparameter("kernel_size", - lower=min_kernel_size, - upper=max_kernel_size, - default_value=kernel_size[1]) - cs.add_hyperparameter(kernel_size_hp) + add_hyperparameter(cs, num_blocks, UniformIntegerHyperparameter) + add_hyperparameter(cs, num_filters, UniformIntegerHyperparameter) + add_hyperparameter(cs, kernel_size, UniformIntegerHyperparameter) + add_hyperparameter(cs, bottleneck_size, UniformIntegerHyperparameter) + return cs diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/MLPBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/MLPBackbone.py index 230ddfe96..2e9d06110 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/MLPBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/MLPBackbone.py @@ -12,6 +12,7 @@ from autoPyTorch.pipeline.components.setup.network_backbone.base_network_backbone import NetworkBackboneComponent from autoPyTorch.pipeline.components.setup.network_backbone.utils import _activations +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter class MLPBackbone(NetworkBackboneComponent): @@ -68,41 +69,50 @@ def get_properties(dataset_properties: Optional[Dict[str, Any]] = None) -> Dict[ } @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, - num_groups: Tuple[Tuple, int] = ((1, 15), 5), - activation: Tuple[Tuple, str] = (tuple(_activations.keys()), - list(_activations.keys())[0]), - use_dropout: Tuple[Tuple, bool] = ((True, False), False), - num_units: Tuple[Tuple, int] = ((10, 1024), 200), - dropout: Tuple[Tuple, float] = ((0, 0.8), 0.5) - ) -> ConfigurationSpace: + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict] = None, + num_groups: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="num_groups", + value_range=(1, 15), + default_value=5, + ), + activation: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="activation", + value_range=tuple(_activations.keys()), + default_value=list(_activations.keys())[0], + ), + use_dropout: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_dropout", + value_range=(True, False), + default_value=False, + ), + num_units: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="num_units", + value_range=(10, 1024), + default_value=200, + ), + dropout: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="dropout", + value_range=(0, 0.8), + default_value=0.5, + ), + ) -> ConfigurationSpace: cs = ConfigurationSpace() # The number of hidden layers the network will have. # Layer blocks are meant to have the same architecture, differing only # by the number of units - min_mlp_layers, max_mlp_layers = num_groups[0] - num_groups = UniformIntegerHyperparameter( - "num_groups", min_mlp_layers, max_mlp_layers, default_value=num_groups[1]) - - activation = CategoricalHyperparameter( - "activation", choices=activation[0], - default_value=activation[1] - ) - cs.add_hyperparameters([num_groups, activation]) + min_mlp_layers, max_mlp_layers = num_groups.value_range + num_groups = get_hyperparameter(num_groups, UniformIntegerHyperparameter) + add_hyperparameter(cs, activation, CategoricalHyperparameter) # We can have dropout in the network for # better generalization - use_dropout = CategoricalHyperparameter( - "use_dropout", choices=use_dropout[0], default_value=use_dropout[1]) - cs.add_hyperparameters([use_dropout]) + use_dropout = get_hyperparameter(use_dropout, CategoricalHyperparameter) + cs.add_hyperparameters([num_groups, use_dropout]) for i in range(1, max_mlp_layers + 1): - n_units_hp = UniformIntegerHyperparameter("num_units_%d" % i, - lower=num_units[0][0], - upper=num_units[0][1], - default_value=num_units[1]) + n_units_search_space = HyperparameterSearchSpace(hyperparameter='num_units_%d' % i, + value_range=num_units.value_range, + default_value=num_units.default_value, + log=num_units.log) + n_units_hp = get_hyperparameter(n_units_search_space, UniformIntegerHyperparameter) cs.add_hyperparameter(n_units_hp) if i > min_mlp_layers: @@ -113,14 +123,13 @@ def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, n_units_hp, num_groups, i - 1 ) ) - - dropout_hp = UniformFloatHyperparameter( - "dropout_%d" % i, - lower=dropout[0][0], - upper=dropout[0][1], - default_value=dropout[1] - ) + dropout_search_space = HyperparameterSearchSpace(hyperparameter='dropout_%d' % i, + value_range=num_units.value_range, + default_value=num_units.default_value, + log=num_units.log) + dropout_hp = get_hyperparameter(dropout_search_space, UniformIntegerHyperparameter) cs.add_hyperparameter(dropout_hp) + dropout_condition_1 = CS.EqualsCondition(dropout_hp, use_dropout, True) if i > min_mlp_layers: diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py index 4433f540c..c179b7c77 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py @@ -19,6 +19,7 @@ shake_get_alpha_beta, shake_shake ) +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter class ResNetBackbone(NetworkBackboneComponent): @@ -92,86 +93,101 @@ def get_properties(dataset_properties: Optional[Dict[str, Any]] = None) -> Dict[ } @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, - num_groups: Tuple[Tuple, int] = ((1, 15), 5), - use_dropout: Tuple[Tuple, bool] = ((True, False), False), - num_units: Tuple[Tuple, int] = ((10, 1024), 200), - activation: Tuple[Tuple, str] = (tuple(_activations.keys()), - list(_activations.keys())[0]), - blocks_per_group: Tuple[Tuple, int] = ((1, 4), 2), - dropout: Tuple[Tuple, float] = ((0, 0.8), 0.5), - use_shake_shake: Tuple[Tuple, bool] = ((True, False), True), - use_shake_drop: Tuple[Tuple, bool] = ((True, False), True), - max_shake_drop_probability: Tuple[Tuple, float] = ((0, 1), 0.5) - ) -> ConfigurationSpace: + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict] = None, + num_groups: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="num_groups", + value_range=(1, 15), + default_value=5, + ), + use_dropout: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_dropout", + value_range=(True, False), + default_value=False, + ), + num_units: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="num_units", + value_range=(10, 1024), + default_value=200, + ), + activation: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="activation", + value_range=tuple(_activations.keys()), + default_value=list(_activations.keys())[0], + ), + blocks_per_group: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="blocks_per_group", + value_range=(1, 4), + default_value=2, + ), + dropout: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="dropout", + value_range=(0, 0.8), + default_value=0.5, + ), + use_shake_shake: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_shake_shake", + value_range=(True, False), + default_value=True, + ), + use_shake_drop: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_shake_drop", + value_range=(True, False), + default_value=True, + ), + max_shake_drop_probability: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="max_shake_drop_probability", + value_range=(0, 1), + default_value=0.5, + ), + ) -> ConfigurationSpace: cs = ConfigurationSpace() # The number of groups that will compose the resnet. That is, # a group can have N Resblock. The M number of this N resblock # repetitions is num_groups - min_num_gropus, max_num_groups = num_groups[0] - num_groups = UniformIntegerHyperparameter( - "num_groups", lower=min_num_gropus, upper=max_num_groups, default_value=num_groups[1]) + min_num_gropus, max_num_groups = num_groups.value_range + num_groups = get_hyperparameter(num_groups, UniformIntegerHyperparameter) - activation = CategoricalHyperparameter( - "activation", choices=activation[0], - default_value=activation[1] - ) - cs.add_hyperparameters([num_groups, activation]) + add_hyperparameter(cs, activation, CategoricalHyperparameter) + cs.add_hyperparameters([num_groups]) # We can have dropout in the network for # better generalization - use_dropout = CategoricalHyperparameter("use_dropout", choices=use_dropout[0], default_value=use_dropout[1]) + use_dropout = get_hyperparameter(use_dropout, CategoricalHyperparameter) cs.add_hyperparameters([use_dropout]) - use_shake_shake = CategoricalHyperparameter("use_shake_shake", choices=use_shake_shake[0], - default_value=use_shake_shake[1]) - use_shake_drop = CategoricalHyperparameter("use_shake_drop", choices=use_shake_drop[0], - default_value=use_shake_drop[1]) - shake_drop_prob = UniformFloatHyperparameter( - "max_shake_drop_probability", - lower=max_shake_drop_probability[0][0], - upper=max_shake_drop_probability[0][1], - default_value=max_shake_drop_probability[1]) + use_shake_shake = get_hyperparameter(use_shake_shake, CategoricalHyperparameter) + use_shake_drop = get_hyperparameter(use_shake_drop, CategoricalHyperparameter) + shake_drop_prob = get_hyperparameter(max_shake_drop_probability, UniformFloatHyperparameter) cs.add_hyperparameters([use_shake_shake, use_shake_drop, shake_drop_prob]) cs.add_condition(CS.EqualsCondition(shake_drop_prob, use_shake_drop, True)) # It is the upper bound of the nr of groups, # since the configuration will actually be sampled. - (min_blocks_per_group, max_blocks_per_group), default_blocks_per_group = blocks_per_group[:2] for i in range(0, max_num_groups + 1): - n_units = UniformIntegerHyperparameter( - "num_units_%d" % i, - lower=num_units[0][0], - upper=num_units[0][1], - default_value=num_units[1] - ) - blocks_per_group = UniformIntegerHyperparameter( - "blocks_per_group_%d" % i, - lower=min_blocks_per_group, - upper=max_blocks_per_group, - default_value=default_blocks_per_group) + n_units_search_space = HyperparameterSearchSpace(hyperparameter='num_units_%d' % i, + value_range=num_units.value_range, + default_value=num_units.default_value, + log=num_units.log) + n_units_hp = get_hyperparameter(n_units_search_space, UniformIntegerHyperparameter) - cs.add_hyperparameters([n_units, blocks_per_group]) + blocks_per_group_search_space = HyperparameterSearchSpace(hyperparameter='blocks_per_group_%d' % i, + value_range=blocks_per_group.value_range, + default_value=blocks_per_group.default_value, + log=blocks_per_group.log) + blocks_per_group_hp = get_hyperparameter(blocks_per_group_search_space, UniformIntegerHyperparameter) + cs.add_hyperparameters([n_units_hp, blocks_per_group_hp]) if i > 1: - cs.add_condition(CS.GreaterThanCondition(n_units, num_groups, i - 1)) - cs.add_condition(CS.GreaterThanCondition(blocks_per_group, num_groups, i - 1)) - - this_dropout = UniformFloatHyperparameter( - "dropout_%d" % i, - lower=dropout[0][0], - upper=dropout[0][1], - default_value=dropout[1] - ) - cs.add_hyperparameters([this_dropout]) + cs.add_condition(CS.GreaterThanCondition(n_units_hp, num_groups, i - 1)) + cs.add_condition(CS.GreaterThanCondition(blocks_per_group_hp, num_groups, i - 1)) + + dropout_search_space = HyperparameterSearchSpace(hyperparameter='dropout_%d' % i, + value_range=dropout.value_range, + default_value=dropout.default_value, + log=dropout.log) + dropout_hp = get_hyperparameter(dropout_search_space, UniformFloatHyperparameter) + cs.add_hyperparameter(dropout_hp) - dropout_condition_1 = CS.EqualsCondition(this_dropout, use_dropout, True) + dropout_condition_1 = CS.EqualsCondition(dropout_hp, use_dropout, True) if i > 1: - dropout_condition_2 = CS.GreaterThanCondition(this_dropout, num_groups, i - 1) + dropout_condition_2 = CS.GreaterThanCondition(dropout_hp, num_groups, i - 1) cs.add_condition(CS.AndConjunction(dropout_condition_1, dropout_condition_2)) else: @@ -185,14 +201,14 @@ class ResBlock(nn.Module): """ def __init__( - self, - config: Dict[str, Any], - in_features: int, - out_features: int, - blocks_per_group: int, - block_index: int, - dropout: bool, - activation: nn.Module + self, + config: Dict[str, Any], + in_features: int, + out_features: int, + blocks_per_group: int, + block_index: int, + dropout: bool, + activation: nn.Module ): super(ResBlock, self).__init__() self.config = config diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/ShapedMLPBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/ShapedMLPBackbone.py index 3e8be6b70..c92b6696d 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/ShapedMLPBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/ShapedMLPBackbone.py @@ -15,6 +15,7 @@ _activations, get_shaped_neuron_counts, ) +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter class ShapedMLPBackbone(NetworkBackboneComponent): @@ -75,57 +76,58 @@ def get_properties(dataset_properties: Optional[Dict[str, Any]] = None) -> Dict[ } @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, - num_groups: Tuple[Tuple, int] = ((1, 15), 5), - max_dropout: Tuple[Tuple, float] = ((0, 1), 0.5), - use_dropout: Tuple[Tuple, bool] = ((True, False), False), - max_units: Tuple[Tuple, int] = ((10, 1024), 200), - output_dim: Tuple[Tuple, int] = ((10, 1024), 200), - mlp_shape: Tuple[Tuple, str] = (('funnel', 'long_funnel', - 'diamond', 'hexagon', - 'brick', 'triangle', 'stairs'), 'funnel'), - activation: Tuple[Tuple, str] = ( - tuple(_activations.keys()), list(_activations.keys())[0]) - ) -> ConfigurationSpace: + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict] = None, + num_groups: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="num_groups", + value_range=(1, 15), + default_value=5, + ), + max_dropout: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="max_dropout", + value_range=(0, 1), + default_value=0.5, + ), + use_dropout: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_dropout", + value_range=(True, False), + default_value=False, + ), + max_units: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="max_units", + value_range=(10, 1024), + default_value=200, + ), + output_dim: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="output_dim", + value_range=(10, 1024), + default_value=200, + ), + mlp_shape: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="mlp_shape", + value_range=('funnel', 'long_funnel', + 'diamond', 'hexagon', + 'brick', 'triangle', + 'stairs'), + default_value='funnel', + ), + activation: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="activation", + value_range=tuple(_activations.keys()), + default_value=list(_activations.keys())[0], + ), + + ) -> ConfigurationSpace: cs = ConfigurationSpace() # The number of groups that will compose the resnet. That is, # a group can have N Resblock. The M number of this N resblock # repetitions is num_groups - num_groups = UniformIntegerHyperparameter( - "num_groups", lower=num_groups[0][0], upper=num_groups[0][1], default_value=num_groups[1]) - - mlp_shape = CategoricalHyperparameter('mlp_shape', choices=mlp_shape[0], - default_value=mlp_shape[1]) - - activation = CategoricalHyperparameter( - "activation", choices=activation[0], - default_value=activation[1] - ) - (min_num_units, max_num_units), default_units = max_units[:2] - max_units = UniformIntegerHyperparameter( - "max_units", - lower=min_num_units, - upper=max_num_units, - default_value=default_units, - ) - - output_dim = UniformIntegerHyperparameter( - "output_dim", - lower=output_dim[0][0], - upper=output_dim[0][1], - default_value=output_dim[1] - ) - - cs.add_hyperparameters([num_groups, activation, mlp_shape, max_units, output_dim]) + add_hyperparameter(cs, num_groups, UniformIntegerHyperparameter) + add_hyperparameter(cs, mlp_shape, CategoricalHyperparameter) + add_hyperparameter(cs, activation, CategoricalHyperparameter) + add_hyperparameter(cs, max_units, UniformIntegerHyperparameter) + add_hyperparameter(cs, output_dim, UniformIntegerHyperparameter) # We can have dropout in the network for # better generalization - use_dropout = CategoricalHyperparameter( - "use_dropout", choices=use_dropout[0], default_value=use_dropout[1]) - max_dropout = UniformFloatHyperparameter("max_dropout", lower=max_dropout[0][0], upper=max_dropout[0][1], - default_value=max_dropout[1]) + use_dropout = get_hyperparameter(use_dropout, CategoricalHyperparameter) + max_dropout = get_hyperparameter(max_dropout, UniformFloatHyperparameter) + cs.add_hyperparameters([use_dropout, max_dropout]) cs.add_condition(CS.EqualsCondition(max_dropout, use_dropout, True)) diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py index fbfae28ad..fcae15377 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py @@ -15,6 +15,7 @@ _activations, get_shaped_neuron_counts, ) +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter class ShapedResNetBackbone(ResNetBackbone): @@ -80,81 +81,77 @@ def get_properties(dataset_properties: Optional[Dict[str, Any]] = None) -> Dict[ } @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, # type: ignore[override] - num_groups: Tuple[Tuple, int] = ((1, 15), 5), - use_dropout: Tuple[Tuple, bool] = ((True, False), False), - max_units: Tuple[Tuple, int] = ((10, 1024), 200), - blocks_per_group: Tuple[Tuple, int] = ((1, 4), 2), - max_dropout: Tuple[Tuple, float] = ((0, 0.8), 0.5), - use_shake_shake: Tuple[Tuple, bool] = ((True, False), True), - use_shake_drop: Tuple[Tuple, bool] = ((True, False), True), - max_shake_drop_probability: Tuple[Tuple, float] = ((0, 1), 0.5), - resnet_shape: Tuple[Tuple, str] = (('funnel', 'long_funnel', - 'diamond', 'hexagon', - 'brick', 'triangle', 'stairs'), 'funnel'), - activation: Tuple[Tuple, str] = ( - tuple(_activations.keys()), list(_activations.keys())[0]), - output_dim: Tuple[Tuple, int] = ((10, 1024), 200), - ) -> ConfigurationSpace: + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict] = None, # type: ignore[override] + resnet_shape: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="resnet_shape", + value_range=('funnel', 'long_funnel', + 'diamond', 'hexagon', + 'brick', 'triangle', + 'stairs'), + default_value='funnel', + ), + output_dim: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="output_dim", + value_range=(10, 1024), + default_value=200, + ), + num_groups: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="num_groups", + value_range=(1, 15), + default_value=5, + ), + use_dropout: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_dropout", + value_range=(True, False), + default_value=False, + ), + max_units: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="max_units", + value_range=(10, 1024), + default_value=200), + activation: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="activation", + value_range=tuple(_activations.keys()), + default_value=list(_activations.keys())[0]), + blocks_per_group: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="blocks_per_group", + value_range=(1, 4), + default_value=2), + max_dropout: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="max_dropout", + value_range=(0, 0.8), + default_value=0.5), + use_shake_shake: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_shake_shake", + value_range=(True, False), + default_value=True), + use_shake_drop: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_shake_drop", + value_range=(True, False), + default_value=True), + max_shake_drop_probability: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="max_shake_drop_probability", + value_range=(0, 1), + default_value=0.5, + ), + ) -> ConfigurationSpace: + cs = ConfigurationSpace() # Support for different shapes - resnet_shape = CategoricalHyperparameter( - 'resnet_shape', - choices=resnet_shape[0], - default_value=resnet_shape[1] - ) - cs.add_hyperparameter(resnet_shape) + add_hyperparameter(cs, resnet_shape, CategoricalHyperparameter) # The number of groups that will compose the resnet. That is, # a group can have N Resblock. The M number of this N resblock # repetitions is num_groups - num_groups = UniformIntegerHyperparameter( - "num_groups", lower=num_groups[0][0], upper=num_groups[0][1], default_value=num_groups[1]) + add_hyperparameter(cs, num_groups, UniformIntegerHyperparameter) + add_hyperparameter(cs, blocks_per_group, UniformIntegerHyperparameter) - blocks_per_group = UniformIntegerHyperparameter( - "blocks_per_group", lower=blocks_per_group[0][0], - upper=blocks_per_group[0][1], - default_value=blocks_per_group[1]) + add_hyperparameter(cs, activation, CategoricalHyperparameter) + add_hyperparameter(cs, output_dim, UniformIntegerHyperparameter) - activation = CategoricalHyperparameter( - "activation", choices=activation[0], - default_value=activation[1] - ) - (min_num_units, max_num_units), default_units = max_units[:2] - output_dim = UniformIntegerHyperparameter( - "output_dim", - lower=output_dim[0][0], - upper=output_dim[0][1], - default_value=output_dim[1] - ) - - cs.add_hyperparameters([num_groups, blocks_per_group, activation, output_dim]) - - use_shake_shake = CategoricalHyperparameter("use_shake_shake", choices=use_shake_shake[0], - default_value=use_shake_shake[1]) - use_shake_drop = CategoricalHyperparameter("use_shake_drop", choices=use_shake_drop[0], - default_value=use_shake_drop[1]) - shake_drop_prob = UniformFloatHyperparameter( - "max_shake_drop_probability", - lower=max_shake_drop_probability[0][0], - upper=max_shake_drop_probability[0][1], - default_value=max_shake_drop_probability[1]) + use_shake_shake = get_hyperparameter(use_shake_shake, CategoricalHyperparameter) + use_shake_drop = get_hyperparameter(use_shake_drop, CategoricalHyperparameter) + shake_drop_prob = get_hyperparameter(max_shake_drop_probability, UniformFloatHyperparameter) cs.add_hyperparameters([use_shake_shake, use_shake_drop, shake_drop_prob]) cs.add_condition(CS.EqualsCondition(shake_drop_prob, use_shake_drop, True)) - max_units = UniformIntegerHyperparameter( - "max_units", - lower=min_num_units, - upper=max_num_units, - default_value=default_units - ) - cs.add_hyperparameters([max_units]) + add_hyperparameter(cs, max_units, UniformIntegerHyperparameter) + + use_dropout = get_hyperparameter(use_dropout, CategoricalHyperparameter) + max_dropout = get_hyperparameter(max_dropout, UniformFloatHyperparameter) - use_dropout = CategoricalHyperparameter( - "use_dropout", choices=use_dropout[0], default_value=use_dropout[1]) - max_dropout = UniformFloatHyperparameter("max_dropout", lower=max_dropout[0][0], upper=max_dropout[0][1], - default_value=max_dropout[1]) cs.add_hyperparameters([use_dropout]) cs.add_hyperparameters([max_dropout]) cs.add_condition(CS.EqualsCondition(max_dropout, use_dropout, True)) diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/TCNBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/TCNBackbone.py index c9768153f..f0614a6fe 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/TCNBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/TCNBackbone.py @@ -13,6 +13,7 @@ from torch.nn.utils import weight_norm from autoPyTorch.pipeline.components.setup.network_backbone.base_network_backbone import NetworkBackboneComponent +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter # _Chomp1d, _TemporalBlock and _TemporalConvNet copied from @@ -122,48 +123,45 @@ def get_properties(dataset_properties: Optional[Dict[str, str]] = None) -> Dict[ } @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, - num_blocks: Tuple[Tuple, int] = ((1, 10), 5), - num_filters: Tuple[Tuple, int] = ((4, 64), 32), - kernel_size: Tuple[Tuple, int] = ((4, 64), 32), - use_dropout: Tuple[Tuple, bool] = ((True, False), False), - dropout: Tuple[Tuple, float] = ((0.0, 0.5), 0.1) - ) -> ConfigurationSpace: + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict] = None, + num_blocks: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="num_blocks", + value_range=(1, 10), + default_value=5), + num_filters: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="num_filters", + value_range=(4, 64), + default_value=32), + kernel_size: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="kernel_size", + value_range=(4, 64), + default_value=32), + use_dropout: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_dropout", + value_range=(True, False), + default_value=False), + dropout: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="dropout", + value_range=(0, 0.5), + default_value=0.1), + ) -> ConfigurationSpace: cs = ConfigurationSpace() - min_num_blocks, max_num_blocks = num_blocks[0] - num_blocks_hp = UniformIntegerHyperparameter("num_blocks", - lower=min_num_blocks, - upper=max_num_blocks, - default_value=num_blocks[1]) + min_num_blocks, max_num_blocks = num_blocks.value_range + num_blocks_hp = get_hyperparameter(num_blocks, UniformIntegerHyperparameter) cs.add_hyperparameter(num_blocks_hp) - min_kernel_size, max_kernel_size = kernel_size[0] - kernel_size_hp = UniformIntegerHyperparameter("kernel_size", - lower=min_kernel_size, - upper=max_kernel_size, - default_value=kernel_size[1]) - cs.add_hyperparameter(kernel_size_hp) + add_hyperparameter(cs, kernel_size, UniformIntegerHyperparameter) - use_dropout_hp = CategoricalHyperparameter("use_dropout", - choices=use_dropout[0], - default_value=use_dropout[1]) + use_dropout_hp = get_hyperparameter(use_dropout, CategoricalHyperparameter) cs.add_hyperparameter(use_dropout_hp) - min_dropout, max_dropout = dropout[0] - dropout_hp = UniformFloatHyperparameter("dropout", - lower=min_dropout, - upper=max_dropout, - default_value=dropout[1]) + dropout_hp = get_hyperparameter(dropout, UniformFloatHyperparameter) cs.add_hyperparameter(dropout_hp) cs.add_condition(CS.EqualsCondition(dropout_hp, use_dropout_hp, True)) for i in range(0, max_num_blocks): - min_num_filters, max_num_filters = num_filters[0] - num_filters_hp = UniformIntegerHyperparameter(f"num_filters_{i}", - lower=min_num_filters, - upper=max_num_filters, - default_value=num_filters[1]) + num_filter_search_space = HyperparameterSearchSpace(f"num_filter_{i}", + value_range=num_filters.value_range, + default_value=num_filters.default_value, + log=num_filters.log) + num_filters_hp = get_hyperparameter(num_filter_search_space, UniformIntegerHyperparameter) cs.add_hyperparameter(num_filters_hp) if i >= min_num_blocks: cs.add_condition(CS.GreaterThanCondition( diff --git a/autoPyTorch/pipeline/components/setup/network_embedding/LearnedEntityEmbedding.py b/autoPyTorch/pipeline/components/setup/network_embedding/LearnedEntityEmbedding.py index 3910afc37..1c41b3000 100644 --- a/autoPyTorch/pipeline/components/setup/network_embedding/LearnedEntityEmbedding.py +++ b/autoPyTorch/pipeline/components/setup/network_embedding/LearnedEntityEmbedding.py @@ -12,6 +12,7 @@ from torch import nn from autoPyTorch.pipeline.components.setup.network_embedding.base_network_embedding import NetworkEmbeddingComponent +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter class _LearnedEntityEmbedding(nn.Module): @@ -100,25 +101,24 @@ def build_embedding(self, num_input_features: np.ndarray, num_numerical_features @staticmethod def get_hyperparameter_search_space( dataset_properties: Optional[Dict[str, str]] = None, - min_unique_values_for_embedding: Tuple[Tuple, int, bool] = ((3, 7), 5, True), - dimension_reduction: Tuple[Tuple, float] = ((0, 1), 0.5), + min_unique_values_for_embedding: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="min_unique_values_for_embedding", + value_range=(3, 7), + default_value=5, + log=True), + dimension_reduction: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="dimension_reduction", + value_range=(0, 1), + default_value=0.5), ) -> ConfigurationSpace: cs = ConfigurationSpace() - min_hp = UniformIntegerHyperparameter("min_unique_values_for_embedding", - lower=min_unique_values_for_embedding[0][0], - upper=min_unique_values_for_embedding[0][1], - default_value=min_unique_values_for_embedding[1], - log=min_unique_values_for_embedding[2] - ) - cs.add_hyperparameter(min_hp) + add_hyperparameter(cs, min_unique_values_for_embedding, UniformIntegerHyperparameter) if dataset_properties is not None: for i in range(len(dataset_properties['categorical_columns'])): - ee_dimensions_hp = UniformFloatHyperparameter("dimension_reduction_" + str(i), - lower=dimension_reduction[0][0], - upper=dimension_reduction[0][1], - default_value=dimension_reduction[1] - ) - cs.add_hyperparameter(ee_dimensions_hp) + ee_dimensions_search_space = HyperparameterSearchSpace(hyperparameter="dimension_reduction_" + str(i), + value_range=dimension_reduction.value_range, + default_value=dimension_reduction.default_value, + log=dimension_reduction.log) + add_hyperparameter(cs, ee_dimensions_search_space, UniformFloatHyperparameter) return cs @staticmethod diff --git a/autoPyTorch/pipeline/components/setup/network_head/fully_connected.py b/autoPyTorch/pipeline/components/setup/network_head/fully_connected.py index f01839234..134d7a27b 100644 --- a/autoPyTorch/pipeline/components/setup/network_head/fully_connected.py +++ b/autoPyTorch/pipeline/components/setup/network_head/fully_connected.py @@ -10,6 +10,7 @@ from autoPyTorch.pipeline.components.setup.network_head.base_network_head import NetworkHeadComponent from autoPyTorch.pipeline.components.setup.network_head.utils import _activations +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter class FullyConnectedHead(NetworkHeadComponent): @@ -42,36 +43,36 @@ def get_properties(dataset_properties: Optional[Dict[str, Any]] = None) -> Dict[ } @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict[str, str]] = None, - num_layers: Tuple[Tuple, int] = ((1, 4), 2), - units_layer: Tuple[Tuple, int] = ((64, 512), 128), - activation: Tuple[Tuple, str] = (tuple(_activations.keys()), - list(_activations.keys())[0]) - ) -> ConfigurationSpace: + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict[str, str]] = None, + num_layers: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="num_layers", + value_range=(1, 4), + default_value=2), + units_layer: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="units_layer", + value_range=(64, 512), + default_value=128), + activation: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="activation", + value_range=tuple(_activations.keys()), + default_value=list(_activations.keys())[0]), + ) -> ConfigurationSpace: cs = ConfigurationSpace() - min_num_layers, max_num_layers = num_layers[0] - num_layers_hp = UniformIntegerHyperparameter("num_layers", - lower=min_num_layers, - upper=max_num_layers, - default_value=num_layers[1] - ) + min_num_layers, max_num_layers = num_layers.value_range + num_layers_hp = get_hyperparameter(num_layers, UniformIntegerHyperparameter) - activation_hp = CategoricalHyperparameter( - "activation", choices=activation[0], - default_value=activation[1] - ) + activation_hp = get_hyperparameter(activation, CategoricalHyperparameter) cs.add_hyperparameters([num_layers_hp, activation_hp]) cs.add_condition(CS.GreaterThanCondition(activation_hp, num_layers_hp, 1)) for i in range(1, max_num_layers): - - num_units_hp = UniformIntegerHyperparameter(f"units_layer_{i}", - lower=units_layer[0][0], - upper=units_layer[0][1], - default_value=units_layer[1]) + num_units_search_space = HyperparameterSearchSpace(hyperparameter=f"units_layer_{i}", + value_range=units_layer.value_range, + default_value=units_layer.default_value, + log=units_layer.log) + num_units_hp = get_hyperparameter(num_units_search_space, UniformIntegerHyperparameter) cs.add_hyperparameter(num_units_hp) + if i >= min_num_layers: cs.add_condition(CS.GreaterThanCondition(num_units_hp, num_layers_hp, i)) diff --git a/autoPyTorch/pipeline/components/setup/network_head/fully_convolutional.py b/autoPyTorch/pipeline/components/setup/network_head/fully_convolutional.py index 21ae3eb71..9388dd774 100644 --- a/autoPyTorch/pipeline/components/setup/network_head/fully_convolutional.py +++ b/autoPyTorch/pipeline/components/setup/network_head/fully_convolutional.py @@ -9,6 +9,7 @@ from autoPyTorch.pipeline.components.setup.network_head.base_network_head import NetworkHeadComponent from autoPyTorch.pipeline.components.setup.network_head.utils import _activations +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter class _FullyConvolutional2DHead(nn.Module): @@ -70,30 +71,39 @@ def get_properties(dataset_properties: Optional[Dict[str, Any]] = None) -> Dict[ } @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict[str, str]] = None, - min_num_layers: int = 1, - max_num_layers: int = 4, - min_num_filters: int = 16, - max_num_filters: int = 256) -> ConfigurationSpace: + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict[str, str]] = None, + num_layers: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="num_layers", + value_range=(1, 4), + default_value=2), + num_filters: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="num_filters", + value_range=(16, 256), + default_value=32), + activation: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="activation", + value_range=tuple(_activations.keys()), + default_value=list(_activations.keys())[0]), + pooling_method: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="pooling_method", + value_range=("average", "max"), + default_value="max"), + ) -> ConfigurationSpace: cs = ConfigurationSpace() - num_layers_hp = UniformIntegerHyperparameter("num_layers", - lower=min_num_layers, - upper=max_num_layers) + min_num_layers, max_num_layers = num_layers.value_range + num_layers_hp = get_hyperparameter(num_layers, UniformIntegerHyperparameter) - pooling_method_hp = CategoricalHyperparameter("pooling_method", - choices=["average", "max"]) + add_hyperparameter(cs, pooling_method, CategoricalHyperparameter) - activation_hp = CategoricalHyperparameter('activation', - choices=list(_activations.keys())) + activation_hp = get_hyperparameter(activation, CategoricalHyperparameter) - cs.add_hyperparameters([num_layers_hp, pooling_method_hp, activation_hp]) + cs.add_hyperparameters([num_layers_hp, activation_hp]) cs.add_condition(CS.GreaterThanCondition(activation_hp, num_layers_hp, 1)) for i in range(1, max_num_layers): - num_filters_hp = UniformIntegerHyperparameter(f"layer_{i}_filters", - lower=min_num_filters, - upper=max_num_filters) + num_filters_search_space = HyperparameterSearchSpace(f"layer_{i}_filters", + value_range=num_filters.value_range, + default_value=num_filters.default_value, + log=num_filters.log) + num_filters_hp = get_hyperparameter(num_filters_search_space, UniformIntegerHyperparameter) cs.add_hyperparameter(num_filters_hp) if i >= min_num_layers: cs.add_condition(CS.GreaterThanCondition(num_filters_hp, num_layers_hp, i)) diff --git a/autoPyTorch/pipeline/components/setup/network_initializer/base_network_initializer.py b/autoPyTorch/pipeline/components/setup/network_initializer/base_network_initializer.py index 93f478f1d..062cdb50d 100644 --- a/autoPyTorch/pipeline/components/setup/network_initializer/base_network_initializer.py +++ b/autoPyTorch/pipeline/components/setup/network_initializer/base_network_initializer.py @@ -11,7 +11,7 @@ import torch from autoPyTorch.pipeline.components.setup.base_setup import autoPyTorchSetupComponent -from autoPyTorch.utils.common import FitRequirement +from autoPyTorch.utils.common import FitRequirement, HyperparameterSearchSpace, add_hyperparameter class BaseNetworkInitializerComponent(autoPyTorchSetupComponent): @@ -72,16 +72,16 @@ def transform(self, X: Dict[str, Any]) -> Dict[str, Any]: return X @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, - bias_strategy: Tuple[Tuple, str] = (('Zero', 'Normal'), 'Normal') - ) -> ConfigurationSpace: - + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict] = None, + bias_strategy: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="bias_strategy", + value_range=('Zero', 'Normal'), + default_value='Normal') + ) -> ConfigurationSpace: cs = ConfigurationSpace() # The strategy for bias initializations - bias_strategy = CategoricalHyperparameter( - "bias_strategy", choices=bias_strategy[0], default_value=bias_strategy[1]) - cs.add_hyperparameters([bias_strategy]) + add_hyperparameter(cs, bias_strategy, CategoricalHyperparameter) return cs def __str__(self) -> str: diff --git a/autoPyTorch/pipeline/components/setup/optimizer/AdamOptimizer.py b/autoPyTorch/pipeline/components/setup/optimizer/AdamOptimizer.py index aa86e9d0c..09a417075 100644 --- a/autoPyTorch/pipeline/components/setup/optimizer/AdamOptimizer.py +++ b/autoPyTorch/pipeline/components/setup/optimizer/AdamOptimizer.py @@ -10,6 +10,7 @@ from torch.optim import Adam from autoPyTorch.pipeline.components.setup.optimizer.base_optimizer import BaseOptimizerComponent +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter class AdamOptimizer(BaseOptimizerComponent): @@ -32,7 +33,6 @@ def __init__( weight_decay: float, random_state: Optional[np.random.RandomState] = None, ): - super().__init__() self.lr = lr self.beta1 = beta1 @@ -73,28 +73,28 @@ def get_properties(dataset_properties: Optional[Dict[str, Any]] = None) -> Dict[ } @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, - lr: Tuple[Tuple, float, bool] = ((1e-5, 1e-1), 1e-2, True), - beta1: Tuple[Tuple, float] = ((0.85, 0.999), 0.9), - beta2: Tuple[Tuple, float] = ((0.9, 0.9999), 0.9), - weight_decay: Tuple[Tuple, float] = ((0.0, 0.1), 0.0) - ) -> ConfigurationSpace: - + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict] = None, + lr: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="lr", + value_range=(1e-5, 1e-1), + default_value=1e-2, + log=True), + beta1: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="beta1", + value_range=(0.85, 0.999), + default_value=0.9), + beta2: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="beta2", + value_range=(0.9, 0.9999), + default_value=0.9), + weight_decay: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="weight_decay", + value_range=(0.0, 0.1), + default_value=0.0), + ) -> ConfigurationSpace: cs = ConfigurationSpace() # The learning rate for the model - lr = UniformFloatHyperparameter('lr', lower=lr[0][0], upper=lr[0][1], - default_value=lr[1], log=lr[2]) - - beta1 = UniformFloatHyperparameter('beta1', lower=beta1[0][0], upper=beta1[0][1], - default_value=beta1[1]) - - beta2 = UniformFloatHyperparameter('beta2', lower=beta2[0][0], upper=beta2[0][1], - default_value=beta2[1]) - - weight_decay = UniformFloatHyperparameter('weight_decay', lower=weight_decay[0][0], upper=weight_decay[0][1], - default_value=weight_decay[1]) - - cs.add_hyperparameters([lr, beta1, beta2, weight_decay]) + add_hyperparameter(cs, lr, UniformFloatHyperparameter) + add_hyperparameter(cs, beta1, UniformFloatHyperparameter) + add_hyperparameter(cs, beta2, UniformFloatHyperparameter) + add_hyperparameter(cs, weight_decay, UniformFloatHyperparameter) return cs diff --git a/autoPyTorch/pipeline/components/setup/optimizer/AdamWOptimizer.py b/autoPyTorch/pipeline/components/setup/optimizer/AdamWOptimizer.py index b525cdef7..a79e62f34 100644 --- a/autoPyTorch/pipeline/components/setup/optimizer/AdamWOptimizer.py +++ b/autoPyTorch/pipeline/components/setup/optimizer/AdamWOptimizer.py @@ -10,6 +10,7 @@ from torch.optim import AdamW from autoPyTorch.pipeline.components.setup.optimizer.base_optimizer import BaseOptimizerComponent +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter class AdamWOptimizer(BaseOptimizerComponent): @@ -32,7 +33,6 @@ def __init__( weight_decay: float, random_state: Optional[np.random.RandomState] = None, ): - super().__init__() self.lr = lr self.beta1 = beta1 @@ -73,28 +73,28 @@ def get_properties(dataset_properties: Optional[Dict[str, Any]] = None) -> Dict[ } @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, - lr: Tuple[Tuple, float, bool] = ((1e-5, 1e-1), 1e-2, True), - beta1: Tuple[Tuple, float] = ((0.85, 0.999), 0.9), - beta2: Tuple[Tuple, float] = ((0.9, 0.9999), 0.9), - weight_decay: Tuple[Tuple, float] = ((0.0, 0.1), 0.0) - ) -> ConfigurationSpace: - + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict] = None, + lr: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="lr", + value_range=(1e-5, 1e-1), + default_value=1e-2, + log=True), + beta1: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="beta1", + value_range=(0.85, 0.999), + default_value=0.9), + beta2: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="beta2", + value_range=(0.9, 0.9999), + default_value=0.9), + weight_decay: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="weight_decay", + value_range=(0.0, 0.1), + default_value=0.0), + ) -> ConfigurationSpace(): cs = ConfigurationSpace() # The learning rate for the model - lr = UniformFloatHyperparameter('lr', lower=lr[0][0], upper=lr[0][1], - default_value=lr[1], log=lr[2]) - - beta1 = UniformFloatHyperparameter('beta1', lower=beta1[0][0], upper=beta1[0][1], - default_value=beta1[1]) - - beta2 = UniformFloatHyperparameter('beta2', lower=beta2[0][0], upper=beta2[0][1], - default_value=beta2[1]) - - weight_decay = UniformFloatHyperparameter('weight_decay', lower=weight_decay[0][0], upper=weight_decay[0][1], - default_value=weight_decay[1]) - - cs.add_hyperparameters([lr, beta1, beta2, weight_decay]) + add_hyperparameter(cs, lr, UniformFloatHyperparameter) + add_hyperparameter(cs, beta1, UniformFloatHyperparameter) + add_hyperparameter(cs, beta2, UniformFloatHyperparameter) + add_hyperparameter(cs, weight_decay, UniformFloatHyperparameter) return cs diff --git a/autoPyTorch/pipeline/components/setup/optimizer/RMSpropOptimizer.py b/autoPyTorch/pipeline/components/setup/optimizer/RMSpropOptimizer.py index e44bb9495..59c2753a7 100644 --- a/autoPyTorch/pipeline/components/setup/optimizer/RMSpropOptimizer.py +++ b/autoPyTorch/pipeline/components/setup/optimizer/RMSpropOptimizer.py @@ -10,6 +10,7 @@ from torch.optim import RMSprop from autoPyTorch.pipeline.components.setup.optimizer.base_optimizer import BaseOptimizerComponent +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter class RMSpropOptimizer(BaseOptimizerComponent): @@ -34,7 +35,6 @@ def __init__( weight_decay: float, random_state: Optional[np.random.RandomState] = None, ): - super().__init__() self.lr = lr self.momentum = momentum @@ -76,28 +76,28 @@ def get_properties(dataset_properties: Optional[Dict[str, Any]] = None) -> Dict[ } @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, - lr: Tuple[Tuple, float, bool] = ((1e-5, 1e-1), 1e-2, True), - alpha: Tuple[Tuple, float] = ((0.1, 0.99), 0.99), - weight_decay: Tuple[Tuple, float] = ((0.0, 0.1), 0.0), - momentum: Tuple[Tuple, float] = ((0.0, 0.99), 0.0), - ) -> ConfigurationSpace: - + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict] = None, + lr: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="lr", + value_range=(1e-5, 1e-1), + default_value=1e-2, + log=True), + alpha: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="alpha", + value_range=(0.1, 0.99), + default_value=0.99), + weight_decay: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="weight_decay", + value_range=(0.0, 0.1), + default_value=0.0), + momentum: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="momentum", + value_range=(0.0, 0.99), + default_value=0.0), + ) -> ConfigurationSpace: cs = ConfigurationSpace() # The learning rate for the model - lr = UniformFloatHyperparameter('lr', lower=lr[0][0], upper=lr[0][1], - default_value=lr[1], log=lr[2]) - - alpha = UniformFloatHyperparameter('alpha', lower=alpha[0][0], upper=alpha[0][1], - default_value=alpha[1]) - - weight_decay = UniformFloatHyperparameter('weight_decay', lower=weight_decay[0][0], upper=weight_decay[0][1], - default_value=weight_decay[1]) - - momentum = UniformFloatHyperparameter('momentum', lower=momentum[0][0], upper=momentum[0][1], - default_value=momentum[1]) - - cs.add_hyperparameters([lr, alpha, weight_decay, momentum]) + add_hyperparameter(cs, lr, UniformFloatHyperparameter) + add_hyperparameter(cs, alpha, UniformFloatHyperparameter) + add_hyperparameter(cs, momentum, UniformFloatHyperparameter) + add_hyperparameter(cs, weight_decay, UniformFloatHyperparameter) return cs diff --git a/autoPyTorch/pipeline/components/setup/optimizer/SGDOptimizer.py b/autoPyTorch/pipeline/components/setup/optimizer/SGDOptimizer.py index 4396cb381..0f0f64fa6 100644 --- a/autoPyTorch/pipeline/components/setup/optimizer/SGDOptimizer.py +++ b/autoPyTorch/pipeline/components/setup/optimizer/SGDOptimizer.py @@ -10,6 +10,7 @@ from torch.optim import SGD from autoPyTorch.pipeline.components.setup.optimizer.base_optimizer import BaseOptimizerComponent +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter class SGDOptimizer(BaseOptimizerComponent): @@ -70,24 +71,25 @@ def get_properties(dataset_properties: Optional[Dict[str, Any]] = None) -> Dict[ } @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, - lr: Tuple[Tuple, float, bool] = ((1e-5, 1e-1), 1e-2, True), - weight_decay: Tuple[Tuple, float] = ((0.0, 0.1), 0.0), - momentum: Tuple[Tuple, float] = ((0.0, 0.99), 0.0), - ) -> ConfigurationSpace: + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict] = None, + lr: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="lr", + value_range=(1e-5, 1e-1), + default_value=1e-2, + log=True), + weight_decay: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="weight_decay", + value_range=(0.0, 0.1), + default_value=0.0), + momentum: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="momentum", + value_range=(0.0, 0.99), + default_value=0.0), + ) -> ConfigurationSpace: cs = ConfigurationSpace() # The learning rate for the model - lr = UniformFloatHyperparameter('lr', lower=lr[0][0], upper=lr[0][1], - default_value=lr[1], log=lr[2]) - - weight_decay = UniformFloatHyperparameter('weight_decay', lower=weight_decay[0][0], upper=weight_decay[0][1], - default_value=weight_decay[1]) - - momentum = UniformFloatHyperparameter('momentum', lower=momentum[0][0], upper=momentum[0][1], - default_value=momentum[1]) - - cs.add_hyperparameters([lr, weight_decay, momentum]) + add_hyperparameter(cs, lr, UniformFloatHyperparameter) + add_hyperparameter(cs, momentum, UniformFloatHyperparameter) + add_hyperparameter(cs, weight_decay, UniformFloatHyperparameter) return cs diff --git a/autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py b/autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py index 95a4fccb2..85557b2e2 100644 --- a/autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py +++ b/autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py @@ -11,11 +11,15 @@ import torchvision - from autoPyTorch.datasets.base_dataset import BaseDataset from autoPyTorch.pipeline.components.training.base_training import autoPyTorchTrainingComponent from autoPyTorch.utils.backend import Backend -from autoPyTorch.utils.common import FitRequirement, custom_collate_fn +from autoPyTorch.utils.common import ( + FitRequirement, + HyperparameterSearchSpace, + add_hyperparameter, + custom_collate_fn +) class BaseDataLoaderComponent(autoPyTorchTrainingComponent): @@ -202,9 +206,9 @@ def check_requirements(self, X: Dict[str, Any], y: Any = None) -> None: if 'split_id' not in X: raise ValueError("To fit a data loader, expected fit dictionary to have split_id. " "Currently X={}.".format( - X - ) - ) + X + ) + ) if 'backend' not in X: raise ValueError("backend is needed to load the data from disk") @@ -248,13 +252,14 @@ def get_torchvision_datasets(self) -> Dict[str, torchvision.datasets.VisionDatas } @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, - batch_size: Tuple[Tuple, int] = ((32, 320), 64) - ) -> ConfigurationSpace: - batch_size = UniformIntegerHyperparameter( - "batch_size", batch_size[0][0], batch_size[0][1], default_value=batch_size[1]) + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict] = None, + batch_size: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="batch_size", + value_range=(32, 320), + default_value=64) + ) -> ConfigurationSpace: cs = ConfigurationSpace() - cs.add_hyperparameters([batch_size]) + add_hyperparameter(cs, batch_size, UniformIntegerHyperparameter) return cs def __str__(self) -> str: diff --git a/autoPyTorch/pipeline/components/training/trainer/MixUpTrainer.py b/autoPyTorch/pipeline/components/training/trainer/MixUpTrainer.py index ef31c27c5..f001689ea 100644 --- a/autoPyTorch/pipeline/components/training/trainer/MixUpTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/MixUpTrainer.py @@ -12,6 +12,7 @@ from autoPyTorch.constants import CLASSIFICATION_TASKS, STRING_TO_TASK_TYPES from autoPyTorch.pipeline.components.training.trainer.base_trainer import BaseTrainerComponent +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter class MixUpTrainer(BaseTrainerComponent): @@ -65,19 +66,19 @@ def get_properties(dataset_properties: typing.Optional[typing.Dict[str, typing.A } @staticmethod - def get_hyperparameter_search_space(dataset_properties: typing.Optional[typing.Dict] = None, - alpha: typing.Tuple[typing.Tuple[float, float], float] = ((0, 1), 0.2), - weighted_loss: typing.Tuple[typing.Tuple, bool] = ((True, False), True) - ) -> ConfigurationSpace: - alpha = UniformFloatHyperparameter( - "alpha", alpha[0][0], alpha[0][1], default_value=alpha[1]) + def get_hyperparameter_search_space( + dataset_properties: typing.Optional[typing.Dict] = None, + alpha: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="alpha", + value_range=(0, 1), + default_value=0.2), + weighted_loss: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="weighted_loss", + value_range=(True, False), + default_value=True), + ) -> ConfigurationSpace: cs = ConfigurationSpace() - cs.add_hyperparameters([alpha]) + add_hyperparameter(cs, alpha, UniformFloatHyperparameter) if dataset_properties is not None: if STRING_TO_TASK_TYPES[dataset_properties['task_type']] in CLASSIFICATION_TASKS: - weighted_loss = CategoricalHyperparameter("weighted_loss", - choices=weighted_loss[0], - default_value=weighted_loss[1]) - cs.add_hyperparameter(weighted_loss) + add_hyperparameter(cs, weighted_loss, CategoricalHyperparameter) return cs diff --git a/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py b/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py index 6acfb2982..e3301a2d2 100644 --- a/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py @@ -7,6 +7,7 @@ from autoPyTorch.constants import CLASSIFICATION_TASKS, STRING_TO_TASK_TYPES from autoPyTorch.pipeline.components.training.trainer.base_trainer import BaseTrainerComponent +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter class StandardTrainer(BaseTrainerComponent): @@ -53,15 +54,15 @@ def get_properties(dataset_properties: typing.Optional[typing.Dict[str, typing.A } @staticmethod - def get_hyperparameter_search_space(dataset_properties: typing.Optional[typing.Dict] = None, - weighted_loss: typing.Tuple[typing.Tuple, bool] = ((True, False), True) - ) -> ConfigurationSpace: + def get_hyperparameter_search_space( + dataset_properties: typing.Optional[typing.Dict] = None, + weighted_loss: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="weighted_loss", + value_range=(True, False), + default_value=True), + ) -> ConfigurationSpace: cs = ConfigurationSpace() if dataset_properties is not None: if STRING_TO_TASK_TYPES[dataset_properties['task_type']] in CLASSIFICATION_TASKS: - weighted_loss = CategoricalHyperparameter("weighted_loss", - choices=weighted_loss[0], - default_value=weighted_loss[1]) - cs.add_hyperparameter(weighted_loss) + add_hyperparameter(cs, weighted_loss, CategoricalHyperparameter) return cs diff --git a/autoPyTorch/utils/common.py b/autoPyTorch/utils/common.py index dbccedac2..6e48929d2 100644 --- a/autoPyTorch/utils/common.py +++ b/autoPyTorch/utils/common.py @@ -52,7 +52,7 @@ class HyperparameterSearchSpace(NamedTuple): Attributes: hyperparameter (str): name of the hyperparameter - value_range (Iterable[HyperparameterValueType]): + value_range (Sequence[HyperparameterValueType]): range of the hyperparameter default_value (HyperparameterValueType): default value of the hyperparameter @@ -62,7 +62,7 @@ class HyperparameterSearchSpace(NamedTuple): hyperparameter: str value_range: Sequence[HyperparameterValueType] default_value: HyperparameterValueType - log: bool + log: bool = False def replace_prefix_in_config_dict(config: Dict[str, Any], prefix: str, replace: str = "") -> Dict[str, Any]: @@ -221,7 +221,7 @@ def get_hyperparameter(hyperparameter: HyperparameterSearchSpace, lower=hyperparameter.value_range[0], upper=hyperparameter.value_range[1], log=hyperparameter.log, - defaul_value=hyperparameter.default_value) + default_value=hyperparameter.default_value) if hyperparameter_type == UniformIntegerHyperparameter: assert len(hyperparameter.value_range) == 2, \ "Int HP range update for %s is specified by the two upper " \ @@ -230,7 +230,7 @@ def get_hyperparameter(hyperparameter: HyperparameterSearchSpace, lower=hyperparameter.value_range[0], upper=hyperparameter.value_range[1], log=hyperparameter.log, - defaul_value=hyperparameter.default_value) + default_value=hyperparameter.default_value) raise ValueError('Unknown type: %s for hp %s' % (hyperparameter_type, hyperparameter.hyperparameter)) @@ -239,6 +239,7 @@ def add_hyperparameter(cs: ConfigurationSpace, hyperparameter_type: Type[Hyperparameter]): """ Adds the given hyperparameter to the given configuration space + Args: cs (ConfigurationSpace): Configuration space where the hyperparameter must be added diff --git a/autoPyTorch/utils/hyperparameter_search_space_update.py b/autoPyTorch/utils/hyperparameter_search_space_update.py index 0ecd22ebe..4be49cb8e 100644 --- a/autoPyTorch/utils/hyperparameter_search_space_update.py +++ b/autoPyTorch/utils/hyperparameter_search_space_update.py @@ -1,9 +1,10 @@ import ast import os +import re from typing import List, Optional, Sequence, Tuple, Union -from autoPyTorch.pipeline.components.base_choice import autoPyTorchChoice -from autoPyTorch.pipeline.components.base_component import autoPyTorchComponent +from sklearn.base import BaseEstimator + from autoPyTorch.utils.common import HyperparameterSearchSpace, HyperparameterValueType @@ -36,11 +37,11 @@ def __init__(self, node_name: str, hyperparameter: str, value_range: Sequence[Hy self.log = log self.default_value = default_value - def apply(self, pipeline: List[Tuple[str, Union[autoPyTorchComponent, autoPyTorchChoice]]]) -> None: + def apply(self, pipeline: List[Tuple[str, BaseEstimator]]) -> None: """ Applies the update to the appropriate hyperparameter of the pipeline Args: - pipeline (List[Tuple[str, Union[autoPyTorchComponent, autoPyTorchChoice]]]): + pipeline (List[Tuple[str, BaseEstimator]]): The named steps of the current autopytorch pipeline Returns: @@ -54,8 +55,23 @@ def __str__(self) -> str: str) else self.default_value, (" log" if self.log else "")) - def get_search_space(self) -> HyperparameterSearchSpace: - return HyperparameterSearchSpace(hyperparameter=self.hyperparameter, + def get_search_space(self, remove_prefix: Optional[str] = None) -> HyperparameterSearchSpace: + """ + Get Update as a HyperparameterSearchSpace object. + + Args: + remove_prefix (Optional[str]): + if specified, remove given prefix from hyperparameter name + + Returns: + HyperparameterSearchSpace + """ + hyperparameter_name = self.hyperparameter + if remove_prefix is not None: + # remove prefix from hyperparameter name + if remove_prefix in self.hyperparameter: + hyperparameter_name = hyperparameter_name.replace(f"{remove_prefix}:", '') + return HyperparameterSearchSpace(hyperparameter=hyperparameter_name, value_range=self.value_range, default_value=self.default_value, log=self.log) @@ -66,12 +82,12 @@ class HyperparameterSearchSpaceUpdates: def __init__(self, updates: Optional[List[HyperparameterSearchSpaceUpdate]] = None) -> None: self.updates = updates if updates is not None else [] - def apply(self, pipeline: List[Tuple[str, Union[autoPyTorchComponent, autoPyTorchChoice]]]) -> None: + def apply(self, pipeline: List[Tuple[str, BaseEstimator]]) -> None: """ Iteratively applies updates to the pipeline Args: - pipeline: (List[Tuple[str, Union[autoPyTorchComponent, autoPyTorchChoice]]]): + pipeline: (List[Tuple[str, BaseEstimator]]): The named steps of the current autoPyTorch pipeline Returns: From 28999a251fbb97a5cd5aa907640042222d4a60b1 Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Mon, 22 Mar 2021 18:13:53 +0100 Subject: [PATCH 03/10] Working search space update with test for __choice__ and fix flake --- .../pipeline/components/base_choice.py | 4 +- .../pipeline/components/base_component.py | 5 +- .../normalise/base_normalizer_choice.py | 19 ++++++-- .../encoding/base_encoder_choice.py | 40 +++++++++++----- .../feature_preprocessing/KernelPCA.py | 2 +- .../feature_preprocessing/Nystroem.py | 2 +- .../PolynomialFeatures.py | 2 +- .../feature_preprocessing/PowerTransformer.py | 13 +++--- .../RandomKitchenSinks.py | 3 +- .../feature_preprocessing/TruncatedSVD.py | 2 +- .../base_feature_preprocessor_choice.py | 46 +++++++++++++------ .../imputation/SimpleImputer.py | 5 +- .../scaling/Normalizer.py | 2 +- .../scaling/base_scaler_choice.py | 40 +++++++++++----- .../setup/augmentation/image/GaussianBlur.py | 4 +- .../setup/augmentation/image/GaussianNoise.py | 4 +- .../setup/augmentation/image/RandomAffine.py | 7 ++- .../setup/augmentation/image/RandomCutout.py | 4 +- .../setup/augmentation/image/Resize.py | 2 +- .../augmentation/image/ZeroPadAndCrop.py | 2 +- .../setup/lr_scheduler/CosineAnnealingLR.py | 2 +- .../CosineAnnealingWarmRestarts.py | 2 +- .../components/setup/lr_scheduler/CyclicLR.py | 2 +- .../setup/lr_scheduler/ExponentialLR.py | 2 +- .../setup/lr_scheduler/ReduceLROnPlateau.py | 2 +- .../components/setup/lr_scheduler/StepLR.py | 2 +- .../lr_scheduler/base_scheduler_choice.py | 26 +++++++---- .../setup/network_backbone/MLPBackbone.py | 2 +- .../setup/network_backbone/ResNetBackbone.py | 3 +- .../network_backbone/ShapedResNetBackbone.py | 19 ++++---- .../setup/network_backbone/TCNBackbone.py | 4 +- .../base_network_backbone_choice.py | 25 +++++++--- .../LearnedEntityEmbedding.py | 2 +- .../base_network_embedding_choice.py | 40 +++++++++++----- .../network_head/base_network_head_choice.py | 25 ++++++---- .../setup/network_head/fully_connected.py | 2 +- .../base_network_init_choice.py | 43 ++++++++++------- .../base_network_initializer.py | 2 +- .../setup/optimizer/AdamOptimizer.py | 2 +- .../setup/optimizer/AdamWOptimizer.py | 2 +- .../setup/optimizer/RMSpropOptimizer.py | 2 +- .../setup/optimizer/SGDOptimizer.py | 2 +- .../setup/optimizer/base_optimizer_choice.py | 25 +++++++--- .../training/data_loader/base_data_loader.py | 7 +-- .../training/trainer/base_trainer_choice.py | 25 +++++++--- .../pipeline/tabular_classification.py | 3 +- autoPyTorch/utils/common.py | 16 ++++--- .../hyperparameter_search_space_update.py | 1 - .../test_tabular_classification.py | 25 +++++++++- 49 files changed, 340 insertions(+), 183 deletions(-) diff --git a/autoPyTorch/pipeline/components/base_choice.py b/autoPyTorch/pipeline/components/base_choice.py index dabdb7a62..0aab76516 100644 --- a/autoPyTorch/pipeline/components/base_choice.py +++ b/autoPyTorch/pipeline/components/base_choice.py @@ -1,7 +1,7 @@ +import re import warnings from collections import OrderedDict -import re -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional from ConfigSpace.configuration_space import Configuration, ConfigurationSpace diff --git a/autoPyTorch/pipeline/components/base_component.py b/autoPyTorch/pipeline/components/base_component.py index f475b9f24..981392671 100644 --- a/autoPyTorch/pipeline/components/base_component.py +++ b/autoPyTorch/pipeline/components/base_component.py @@ -5,7 +5,7 @@ import sys import warnings from collections import OrderedDict -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional from ConfigSpace.configuration_space import Configuration, ConfigurationSpace @@ -36,8 +36,7 @@ def find_components( module = importlib.import_module(full_module_name) for member_name, obj in inspect.getmembers(module): - if inspect.isclass(obj) and issubclass(obj, base_class) and \ - obj != base_class: + if inspect.isclass(obj) and issubclass(obj, base_class) and obj != base_class: # TODO test if the obj implements the interface # Keep in mind that this only instantiates the ensemble_wrapper, # but not the real target classifier diff --git a/autoPyTorch/pipeline/components/preprocessing/image_preprocessing/normalise/base_normalizer_choice.py b/autoPyTorch/pipeline/components/preprocessing/image_preprocessing/normalise/base_normalizer_choice.py index 696601b4f..acc9f5b64 100644 --- a/autoPyTorch/pipeline/components/preprocessing/image_preprocessing/normalise/base_normalizer_choice.py +++ b/autoPyTorch/pipeline/components/preprocessing/image_preprocessing/normalise/base_normalizer_choice.py @@ -76,10 +76,21 @@ def get_hyperparameter_search_space(self, default = default_ break - preprocessor = CSH.CategoricalHyperparameter('__choice__', - list(available_preprocessors.keys()), - default_value=default) - + updates = self._get_search_space_updates() + if '__choice__' in updates.keys(): + choice_hyperparameter = updates['__choice__'] + if not set(choice_hyperparameter.value_range).issubset(available_preprocessors): + raise ValueError("Expected given update for {} to have " + "choices in {} got {}".format(self.__class__.__name__, + available_preprocessors, + choice_hyperparameter.value_range)) + preprocessor = CSH.CategoricalHyperparameter('__choice__', + choice_hyperparameter.value_range, + default_value=choice_hyperparameter.default_value) + else: + preprocessor = CSH.CategoricalHyperparameter('__choice__', + list(available_preprocessors.keys()), + default_value=default) cs.add_hyperparameter(preprocessor) # add only child hyperparameters of early_preprocessor choices diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/encoding/base_encoder_choice.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/encoding/base_encoder_choice.py index df71ff209..7ddbf8eaf 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/encoding/base_encoder_choice.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/encoding/base_encoder_choice.py @@ -75,19 +75,37 @@ def get_hyperparameter_search_space(self, default = default_ break - # add only no encoder to choice hyperparameters in case the dataset is only numerical - if len(dataset_properties['categorical_columns']) == 0: - default = 'NoEncoder' - if include is not None and default not in include: - raise ValueError("Provided {} in include, however, the dataset " - "is incompatible with it".format(include)) + updates = self._get_search_space_updates() + if '__choice__' in updates.keys(): + choice_hyperparameter = updates['__choice__'] + if not set(choice_hyperparameter.value_range).issubset(available_preprocessors): + raise ValueError("Expected given update for {} to have " + "choices in {} got {}".format(self.__class__.__name__, + available_preprocessors, + choice_hyperparameter.value_range)) + if len(dataset_properties['categorical_columns']) == 0: + assert len(choice_hyperparameter.value_range) == 1 + assert 'NoEncoder' in choice_hyperparameter.value_range, \ + "Provided {} in choices, however, the dataset " \ + "is incompatible with it".format(choice_hyperparameter.value_range) + preprocessor = CSH.CategoricalHyperparameter('__choice__', - ['NoEncoder'], - default_value=default) + choice_hyperparameter.value_range, + default_value=choice_hyperparameter.default_value) else: - preprocessor = CSH.CategoricalHyperparameter('__choice__', - list(available_preprocessors.keys()), - default_value=default) + # add only no encoder to choice hyperparameters in case the dataset is only numerical + if len(dataset_properties['categorical_columns']) == 0: + default = 'NoEncoder' + if include is not None and default not in include: + raise ValueError("Provided {} in include, however, the dataset " + "is incompatible with it".format(include)) + preprocessor = CSH.CategoricalHyperparameter('__choice__', + ['NoEncoder'], + default_value=default) + else: + preprocessor = CSH.CategoricalHyperparameter('__choice__', + list(available_preprocessors.keys()), + default_value=default) cs.add_hyperparameter(preprocessor) diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/KernelPCA.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/KernelPCA.py index 8a62ba0ea..ee471a71f 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/KernelPCA.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/KernelPCA.py @@ -1,5 +1,5 @@ from math import ceil, floor -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, Optional, Union from ConfigSpace.conditions import EqualsCondition, InCondition from ConfigSpace.configuration_space import ConfigurationSpace diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/Nystroem.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/Nystroem.py index d0f7a237a..e4e007e61 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/Nystroem.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/Nystroem.py @@ -1,5 +1,5 @@ from math import ceil, floor -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, Optional, Union from ConfigSpace.conditions import EqualsCondition, InCondition from ConfigSpace.configuration_space import ConfigurationSpace diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/PolynomialFeatures.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/PolynomialFeatures.py index dc05ed86f..a41c0a26d 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/PolynomialFeatures.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/PolynomialFeatures.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, Optional, Union from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/PowerTransformer.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/PowerTransformer.py index 3e7a3cf5d..767a0f6c1 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/PowerTransformer.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/PowerTransformer.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, Optional, Union from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( @@ -37,13 +37,12 @@ def get_properties(dataset_properties: Optional[Dict[str, str]] = None) -> Dict[ @staticmethod def get_hyperparameter_search_space( - dataset_properties: Optional[Dict[str, str]] = None, - standardize: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='standardize', - value_range=(True, False), - default_value=True, - ), + dataset_properties: Optional[Dict[str, str]] = None, + standardize: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='standardize', + value_range=(True, False), + default_value=True, + ), ) -> ConfigurationSpace: - cs = ConfigurationSpace() add_hyperparameter(cs, standardize, CategoricalHyperparameter) diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/RandomKitchenSinks.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/RandomKitchenSinks.py index acdad0594..b2643e768 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/RandomKitchenSinks.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/RandomKitchenSinks.py @@ -1,5 +1,5 @@ from math import ceil, floor -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, Optional, Union from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( @@ -16,6 +16,7 @@ base_feature_preprocessor import autoPyTorchFeaturePreprocessingComponent from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter + class RandomKitchenSinks(autoPyTorchFeaturePreprocessingComponent): def __init__(self, n_components: int = 100, gamma: float = 1.0, diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/TruncatedSVD.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/TruncatedSVD.py index 5aeea161d..7fdccced2 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/TruncatedSVD.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/TruncatedSVD.py @@ -1,5 +1,5 @@ from math import floor -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, Optional, Union from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/base_feature_preprocessor_choice.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/base_feature_preprocessor_choice.py index 56af71877..43a1e1a66 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/base_feature_preprocessor_choice.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/base_feature_preprocessor_choice.py @@ -82,23 +82,39 @@ def get_hyperparameter_search_space(self, continue default = default_ break - - # add only no feature preprocessor to choice hyperparameters in case the dataset is only categorical - if len(dataset_properties['numerical_columns']) == 0: - default = 'NoFeaturePreprocessor' - if include is not None and default not in include: - raise ValueError("Provided {} in include, however, " - "the dataset is incompatible with it".format(include)) + updates = self._get_search_space_updates() + if '__choice__' in updates.keys(): + choice_hyperparameter = updates['__choice__'] + if not set(choice_hyperparameter.value_range).issubset(available_): + raise ValueError("Expected given update for {} to have " + "choices in {} got {}".format(self.__class__.__name__, + available_, + choice_hyperparameter.value_range)) + if len(dataset_properties['numerical_columns']) == 0: + assert len(choice_hyperparameter.value_range) == 1 + assert 'NoFeaturePreprocessor' in choice_hyperparameter.value_range, \ + "Provided {} in choices, however, the dataset " \ + "is incompatible with it".format(choice_hyperparameter.value_range) preprocessor = CSH.CategoricalHyperparameter('__choice__', - ['NoFeaturePreprocessor'], - default_value=default) + choice_hyperparameter.value_range, + default_value=choice_hyperparameter.default_value) else: - # Truncated SVD requires n_features > n_components - if len(dataset_properties['numerical_columns']) == 1: - del available_['TruncatedSVD'] - preprocessor = CSH.CategoricalHyperparameter('__choice__', - list(available_.keys()), - default_value=default) + # add only no feature preprocessor to choice hyperparameters in case the dataset is only categorical + if len(dataset_properties['numerical_columns']) == 0: + default = 'NoFeaturePreprocessor' + if include is not None and default not in include: + raise ValueError("Provided {} in include, however, " + "the dataset is incompatible with it".format(include)) + preprocessor = CSH.CategoricalHyperparameter('__choice__', + ['NoFeaturePreprocessor'], + default_value=default) + else: + # Truncated SVD requires n_features > n_components + if len(dataset_properties['numerical_columns']) == 1: + del available_['TruncatedSVD'] + preprocessor = CSH.CategoricalHyperparameter('__choice__', + list(available_.keys()), + default_value=default) cs.add_hyperparameter(preprocessor) diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/imputation/SimpleImputer.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/imputation/SimpleImputer.py index 4db01dd37..6f0c8a2c2 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/imputation/SimpleImputer.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/imputation/SimpleImputer.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, Optional, Union from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( @@ -78,8 +78,7 @@ def get_hyperparameter_search_space( hyperparameter='categorical_strategy', value_range=("most_frequent", "constant_!missing!"), - default_value="most_frequent", - ) + default_value="most_frequent") ) -> ConfigurationSpace: cs = ConfigurationSpace() assert dataset_properties is not None, "To create hyperparameter search space" \ diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/Normalizer.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/Normalizer.py index 962904d70..374188812 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/Normalizer.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/Normalizer.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, Optional, Union from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/base_scaler_choice.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/base_scaler_choice.py index 6fdcc47bc..ec7416005 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/base_scaler_choice.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/base_scaler_choice.py @@ -70,20 +70,36 @@ def get_hyperparameter_search_space(self, if default_ in available_scalers: default = default_ break - - # add only no scaler to choice hyperparameters in case the dataset is only categorical - if len(dataset_properties['numerical_columns']) == 0: - default = 'NoScaler' - if include is not None and default not in include: - raise ValueError("Provided {} in include, however, " - "the dataset is incompatible with it".format(include)) + updates = self._get_search_space_updates() + if '__choice__' in updates.keys(): + choice_hyperparameter = updates['__choice__'] + if not set(choice_hyperparameter.value_range).issubset(available_scalers): + raise ValueError("Expected given update for {} to have " + "choices in {} got {}".format(self.__class__.__name__, + available_scalers, + choice_hyperparameter.value_range)) + if len(dataset_properties['numerical_columns']) == 0: + assert len(choice_hyperparameter.value_range) == 1 + assert 'NoScaler' in choice_hyperparameter.value_range, \ + "Provided {} in choices, however, the dataset " \ + "is incompatible with it".format(choice_hyperparameter.value_range) preprocessor = CSH.CategoricalHyperparameter('__choice__', - ['NoScaler'], - default_value=default) + choice_hyperparameter.value_range, + default_value=choice_hyperparameter.default_value) else: - preprocessor = CSH.CategoricalHyperparameter('__choice__', - list(available_scalers.keys()), - default_value=default) + # add only no scaler to choice hyperparameters in case the dataset is only categorical + if len(dataset_properties['numerical_columns']) == 0: + default = 'NoScaler' + if include is not None and default not in include: + raise ValueError("Provided {} in include, however, " + "the dataset is incompatible with it".format(include)) + preprocessor = CSH.CategoricalHyperparameter('__choice__', + ['NoScaler'], + default_value=default) + else: + preprocessor = CSH.CategoricalHyperparameter('__choice__', + list(available_scalers.keys()), + default_value=default) cs.add_hyperparameter(preprocessor) # add only child hyperparameters of early_preprocessor choices diff --git a/autoPyTorch/pipeline/components/setup/augmentation/image/GaussianBlur.py b/autoPyTorch/pipeline/components/setup/augmentation/image/GaussianBlur.py index f57f00d2d..962d959d8 100644 --- a/autoPyTorch/pipeline/components/setup/augmentation/image/GaussianBlur.py +++ b/autoPyTorch/pipeline/components/setup/augmentation/image/GaussianBlur.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, Optional, Union import ConfigSpace as CS from ConfigSpace.configuration_space import ConfigurationSpace @@ -13,7 +13,7 @@ import numpy as np from autoPyTorch.pipeline.components.setup.augmentation.image.base_image_augmenter import BaseImageAugmenter -from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter +from autoPyTorch.utils.common import HyperparameterSearchSpace, get_hyperparameter class GaussianBlur(BaseImageAugmenter): diff --git a/autoPyTorch/pipeline/components/setup/augmentation/image/GaussianNoise.py b/autoPyTorch/pipeline/components/setup/augmentation/image/GaussianNoise.py index c88dc3136..48178432e 100644 --- a/autoPyTorch/pipeline/components/setup/augmentation/image/GaussianNoise.py +++ b/autoPyTorch/pipeline/components/setup/augmentation/image/GaussianNoise.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, Optional, Union import ConfigSpace as CS from ConfigSpace.configuration_space import ConfigurationSpace @@ -13,7 +13,7 @@ import numpy as np from autoPyTorch.pipeline.components.setup.augmentation.image.base_image_augmenter import BaseImageAugmenter -from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter +from autoPyTorch.utils.common import HyperparameterSearchSpace, get_hyperparameter class GaussianNoise(BaseImageAugmenter): diff --git a/autoPyTorch/pipeline/components/setup/augmentation/image/RandomAffine.py b/autoPyTorch/pipeline/components/setup/augmentation/image/RandomAffine.py index fb19bfbca..3db351ee2 100644 --- a/autoPyTorch/pipeline/components/setup/augmentation/image/RandomAffine.py +++ b/autoPyTorch/pipeline/components/setup/augmentation/image/RandomAffine.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, Optional, Union import ConfigSpace as CS from ConfigSpace.configuration_space import ConfigurationSpace @@ -14,7 +14,7 @@ import numpy as np from autoPyTorch.pipeline.components.setup.augmentation.image.base_image_augmenter import BaseImageAugmenter -from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter +from autoPyTorch.utils.common import HyperparameterSearchSpace, get_hyperparameter class RandomAffine(BaseImageAugmenter): @@ -50,8 +50,7 @@ def get_hyperparameter_search_space( translate_percent_offset: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="translate_percent_offset", value_range=(0, 0.4), - default_value=0.2, - ), + default_value=0.2), shear: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="shear", value_range=(0, 45), default_value=30, diff --git a/autoPyTorch/pipeline/components/setup/augmentation/image/RandomCutout.py b/autoPyTorch/pipeline/components/setup/augmentation/image/RandomCutout.py index b25a594c8..493375ca7 100644 --- a/autoPyTorch/pipeline/components/setup/augmentation/image/RandomCutout.py +++ b/autoPyTorch/pipeline/components/setup/augmentation/image/RandomCutout.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, Optional, Union import ConfigSpace as CS from ConfigSpace.configuration_space import ConfigurationSpace @@ -13,7 +13,7 @@ import numpy as np from autoPyTorch.pipeline.components.setup.augmentation.image.base_image_augmenter import BaseImageAugmenter -from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter +from autoPyTorch.utils.common import HyperparameterSearchSpace, get_hyperparameter class RandomCutout(BaseImageAugmenter): diff --git a/autoPyTorch/pipeline/components/setup/augmentation/image/Resize.py b/autoPyTorch/pipeline/components/setup/augmentation/image/Resize.py index f31fae536..14001a8a7 100644 --- a/autoPyTorch/pipeline/components/setup/augmentation/image/Resize.py +++ b/autoPyTorch/pipeline/components/setup/augmentation/image/Resize.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, Optional, Union from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( diff --git a/autoPyTorch/pipeline/components/setup/augmentation/image/ZeroPadAndCrop.py b/autoPyTorch/pipeline/components/setup/augmentation/image/ZeroPadAndCrop.py index 12f1de85e..df18e48b4 100644 --- a/autoPyTorch/pipeline/components/setup/augmentation/image/ZeroPadAndCrop.py +++ b/autoPyTorch/pipeline/components/setup/augmentation/image/ZeroPadAndCrop.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, Optional, Union from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( diff --git a/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingLR.py b/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingLR.py index d7a5ba727..05351f3aa 100644 --- a/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingLR.py +++ b/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingLR.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple +from typing import Any, Dict, Optional from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( diff --git a/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingWarmRestarts.py b/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingWarmRestarts.py index 72fe63642..6e64af802 100644 --- a/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingWarmRestarts.py +++ b/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingWarmRestarts.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple +from typing import Any, Dict, Optional from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import UniformFloatHyperparameter, UniformIntegerHyperparameter diff --git a/autoPyTorch/pipeline/components/setup/lr_scheduler/CyclicLR.py b/autoPyTorch/pipeline/components/setup/lr_scheduler/CyclicLR.py index 35e881e0a..972134064 100644 --- a/autoPyTorch/pipeline/components/setup/lr_scheduler/CyclicLR.py +++ b/autoPyTorch/pipeline/components/setup/lr_scheduler/CyclicLR.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple +from typing import Any, Dict, Optional from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( diff --git a/autoPyTorch/pipeline/components/setup/lr_scheduler/ExponentialLR.py b/autoPyTorch/pipeline/components/setup/lr_scheduler/ExponentialLR.py index 489d7ba38..177c9d551 100644 --- a/autoPyTorch/pipeline/components/setup/lr_scheduler/ExponentialLR.py +++ b/autoPyTorch/pipeline/components/setup/lr_scheduler/ExponentialLR.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple +from typing import Any, Dict, Optional from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( diff --git a/autoPyTorch/pipeline/components/setup/lr_scheduler/ReduceLROnPlateau.py b/autoPyTorch/pipeline/components/setup/lr_scheduler/ReduceLROnPlateau.py index ffb7d6e96..60711b387 100644 --- a/autoPyTorch/pipeline/components/setup/lr_scheduler/ReduceLROnPlateau.py +++ b/autoPyTorch/pipeline/components/setup/lr_scheduler/ReduceLROnPlateau.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple +from typing import Any, Dict, Optional from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( diff --git a/autoPyTorch/pipeline/components/setup/lr_scheduler/StepLR.py b/autoPyTorch/pipeline/components/setup/lr_scheduler/StepLR.py index eaa578eb1..9f3b97db8 100644 --- a/autoPyTorch/pipeline/components/setup/lr_scheduler/StepLR.py +++ b/autoPyTorch/pipeline/components/setup/lr_scheduler/StepLR.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple +from typing import Any, Dict, Optional from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( diff --git a/autoPyTorch/pipeline/components/setup/lr_scheduler/base_scheduler_choice.py b/autoPyTorch/pipeline/components/setup/lr_scheduler/base_scheduler_choice.py index 0be6aec96..9349ab642 100644 --- a/autoPyTorch/pipeline/components/setup/lr_scheduler/base_scheduler_choice.py +++ b/autoPyTorch/pipeline/components/setup/lr_scheduler/base_scheduler_choice.py @@ -15,7 +15,6 @@ ) from autoPyTorch.pipeline.components.setup.lr_scheduler.base_scheduler import BaseLRComponent - directory = os.path.split(__file__)[0] _schedulers = find_components(__package__, directory, @@ -150,14 +149,25 @@ def get_hyperparameter_search_space( if default_ in available_schedulers: default = default_ break - - scheduler = CSH.CategoricalHyperparameter( - '__choice__', - list(available_schedulers.keys()), - default_value=default - ) + updates = self._get_search_space_updates() + if '__choice__' in updates.keys(): + choice_hyperparameter = updates['__choice__'] + if not set(choice_hyperparameter.value_range).issubset(available_schedulers): + raise ValueError("Expected given update for {} to have " + "choices in {} got {}".format(self.__class__.__name__, + available_schedulers, + choice_hyperparameter.value_range)) + scheduler = CSH.CategoricalHyperparameter('__choice__', + choice_hyperparameter.value_range, + default_value=choice_hyperparameter.default_value) + else: + scheduler = CSH.CategoricalHyperparameter( + '__choice__', + list(available_schedulers.keys()), + default_value=default + ) cs.add_hyperparameter(scheduler) - for name in available_schedulers: + for name in scheduler.choices: updates = self._get_search_space_updates(prefix=name) config_space = available_schedulers[name].get_hyperparameter_search_space(dataset_properties, # type:ignore **updates) diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/MLPBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/MLPBackbone.py index 2e9d06110..ade48155e 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/MLPBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/MLPBackbone.py @@ -127,7 +127,7 @@ def get_hyperparameter_search_space( value_range=num_units.value_range, default_value=num_units.default_value, log=num_units.log) - dropout_hp = get_hyperparameter(dropout_search_space, UniformIntegerHyperparameter) + dropout_hp = get_hyperparameter(dropout_search_space, UniformFloatHyperparameter) cs.add_hyperparameter(dropout_hp) dropout_condition_1 = CS.EqualsCondition(dropout_hp, use_dropout, True) diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py index c179b7c77..d57d783a4 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py @@ -130,8 +130,7 @@ def get_hyperparameter_search_space( max_shake_drop_probability: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="max_shake_drop_probability", value_range=(0, 1), - default_value=0.5, - ), + default_value=0.5), ) -> ConfigurationSpace: cs = ConfigurationSpace() diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py index fcae15377..2988d4c6b 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py @@ -84,12 +84,12 @@ def get_properties(dataset_properties: Optional[Dict[str, Any]] = None) -> Dict[ def get_hyperparameter_search_space( dataset_properties: Optional[Dict] = None, # type: ignore[override] resnet_shape: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="resnet_shape", - value_range=('funnel', 'long_funnel', - 'diamond', 'hexagon', - 'brick', 'triangle', - 'stairs'), - default_value='funnel', - ), + value_range=('funnel', 'long_funnel', + 'diamond', 'hexagon', + 'brick', 'triangle', + 'stairs'), + default_value='funnel', + ), output_dim: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="output_dim", value_range=(10, 1024), default_value=200, @@ -112,8 +112,8 @@ def get_hyperparameter_search_space( value_range=(1, 4), default_value=2), max_dropout: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="max_dropout", - value_range=(0, 0.8), - default_value=0.5), + value_range=(0, 0.8), + default_value=0.5), use_shake_shake: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_shake_shake", value_range=(True, False), default_value=True), @@ -123,8 +123,7 @@ def get_hyperparameter_search_space( max_shake_drop_probability: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="max_shake_drop_probability", value_range=(0, 1), - default_value=0.5, - ), + default_value=0.5), ) -> ConfigurationSpace: cs = ConfigurationSpace() diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/TCNBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/TCNBackbone.py index f0614a6fe..9e9ad5f31 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/TCNBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/TCNBackbone.py @@ -138,8 +138,8 @@ def get_hyperparameter_search_space( value_range=(True, False), default_value=False), dropout: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="dropout", - value_range=(0, 0.5), - default_value=0.1), + value_range=(0, 0.5), + default_value=0.1), ) -> ConfigurationSpace: cs = ConfigurationSpace() diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/base_network_backbone_choice.py b/autoPyTorch/pipeline/components/setup/network_backbone/base_network_backbone_choice.py index 71d8a63d3..13793c393 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/base_network_backbone_choice.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/base_network_backbone_choice.py @@ -158,14 +158,25 @@ def get_hyperparameter_search_space( if default_ in available_backbones: default = default_ break - - backbone = CSH.CategoricalHyperparameter( - '__choice__', - list(available_backbones.keys()), - default_value=default - ) + updates = self._get_search_space_updates() + if '__choice__' in updates.keys(): + choice_hyperparameter = updates['__choice__'] + if not set(choice_hyperparameter.value_range).issubset(available_backbones): + raise ValueError("Expected given update for {} to have " + "choices in {} got {}".format(self.__class__.__name__, + available_backbones, + choice_hyperparameter.value_range)) + backbone = CSH.CategoricalHyperparameter('__choice__', + choice_hyperparameter.value_range, + default_value=choice_hyperparameter.default_value) + else: + backbone = CSH.CategoricalHyperparameter( + '__choice__', + list(available_backbones.keys()), + default_value=default + ) cs.add_hyperparameter(backbone) - for name in available_backbones: + for name in backbone.choices: updates = self._get_search_space_updates(prefix=name) config_space = available_backbones[name].get_hyperparameter_search_space(dataset_properties, # type: ignore **updates) diff --git a/autoPyTorch/pipeline/components/setup/network_embedding/LearnedEntityEmbedding.py b/autoPyTorch/pipeline/components/setup/network_embedding/LearnedEntityEmbedding.py index 1c41b3000..648c63e0c 100644 --- a/autoPyTorch/pipeline/components/setup/network_embedding/LearnedEntityEmbedding.py +++ b/autoPyTorch/pipeline/components/setup/network_embedding/LearnedEntityEmbedding.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, Optional, Union from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( diff --git a/autoPyTorch/pipeline/components/setup/network_embedding/base_network_embedding_choice.py b/autoPyTorch/pipeline/components/setup/network_embedding/base_network_embedding_choice.py index c08b156ce..45d15f06a 100644 --- a/autoPyTorch/pipeline/components/setup/network_embedding/base_network_embedding_choice.py +++ b/autoPyTorch/pipeline/components/setup/network_embedding/base_network_embedding_choice.py @@ -146,26 +146,42 @@ def get_hyperparameter_search_space( if default is None: defaults = [ + 'NoEmbedding', 'LearnedEntityEmbedding', - 'NoEmbedding' ] for default_ in defaults: if default_ in available_embedding: default = default_ break - - if len(dataset_properties['categorical_columns']) == 0: - default = 'NoEmbedding' - if include is not None and default not in include: - raise ValueError("Provided {} in include, however, the dataset " - "is incompatible with it".format(include)) + updates = self._get_search_space_updates() + if '__choice__' in updates.keys(): + choice_hyperparameter = updates['__choice__'] + if not set(choice_hyperparameter.value_range).issubset(available_embedding): + raise ValueError("Expected given update for {} to have " + "choices in {} got {}".format(self.__class__.__name__, + available_embedding, + choice_hyperparameter.value_range)) + if len(dataset_properties['categorical_columns']) == 0: + assert len(choice_hyperparameter.value_range) == 1 + assert 'NoEmbedding' in choice_hyperparameter.value_range, \ + "Provided {} in choices, however, the dataset " \ + "is incompatible with it".format(choice_hyperparameter.value_range) embedding = CSH.CategoricalHyperparameter('__choice__', - ['NoEmbedding'], - default_value=default) + choice_hyperparameter.value_range, + default_value=choice_hyperparameter.default_value) else: - embedding = CSH.CategoricalHyperparameter('__choice__', - list(available_embedding.keys()), - default_value=default) + if len(dataset_properties['categorical_columns']) == 0: + default = 'NoEmbedding' + if include is not None and default not in include: + raise ValueError("Provided {} in include, however, the dataset " + "is incompatible with it".format(include)) + embedding = CSH.CategoricalHyperparameter('__choice__', + ['NoEmbedding'], + default_value=default) + else: + embedding = CSH.CategoricalHyperparameter('__choice__', + list(available_embedding.keys()), + default_value=default) cs.add_hyperparameter(embedding) for name in embedding.choices: diff --git a/autoPyTorch/pipeline/components/setup/network_head/base_network_head_choice.py b/autoPyTorch/pipeline/components/setup/network_head/base_network_head_choice.py index f57e60b91..c03e860fc 100644 --- a/autoPyTorch/pipeline/components/setup/network_head/base_network_head_choice.py +++ b/autoPyTorch/pipeline/components/setup/network_head/base_network_head_choice.py @@ -17,7 +17,6 @@ NetworkHeadComponent, ) - directory = os.path.split(__file__)[0] _heads = find_components(__package__, directory, @@ -157,14 +156,24 @@ def get_hyperparameter_search_space( if default_ in available_heads: default = default_ break - - head = CSH.CategoricalHyperparameter( - '__choice__', - list(available_heads.keys()), - default_value=default - ) + updates = self._get_search_space_updates() + if '__choice__' in updates.keys(): + choice_hyperparameter = updates['__choice__'] + if not set(choice_hyperparameter.value_range).issubset(available_heads): + raise ValueError("Expected given update for {} to have " + "choices in {} got {}".format(self.__class__.__name__, + available_heads, + choice_hyperparameter.value_range)) + head = CSH.CategoricalHyperparameter('__choice__', + choice_hyperparameter.value_range, + default_value=choice_hyperparameter.default_value) + else: + head = CSH.CategoricalHyperparameter( + '__choice__', + list(available_heads.keys()), + default_value=default) cs.add_hyperparameter(head) - for name in available_heads: + for name in head.choices: updates = self._get_search_space_updates(prefix=name) config_space = available_heads[name].get_hyperparameter_search_space(dataset_properties, # type: ignore **updates) diff --git a/autoPyTorch/pipeline/components/setup/network_head/fully_connected.py b/autoPyTorch/pipeline/components/setup/network_head/fully_connected.py index 134d7a27b..ebfabdb92 100644 --- a/autoPyTorch/pipeline/components/setup/network_head/fully_connected.py +++ b/autoPyTorch/pipeline/components/setup/network_head/fully_connected.py @@ -10,7 +10,7 @@ from autoPyTorch.pipeline.components.setup.network_head.base_network_head import NetworkHeadComponent from autoPyTorch.pipeline.components.setup.network_head.utils import _activations -from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter +from autoPyTorch.utils.common import HyperparameterSearchSpace, get_hyperparameter class FullyConnectedHead(NetworkHeadComponent): diff --git a/autoPyTorch/pipeline/components/setup/network_initializer/base_network_init_choice.py b/autoPyTorch/pipeline/components/setup/network_initializer/base_network_init_choice.py index ea9b4c1d9..cc7dfcfc6 100644 --- a/autoPyTorch/pipeline/components/setup/network_initializer/base_network_init_choice.py +++ b/autoPyTorch/pipeline/components/setup/network_initializer/base_network_init_choice.py @@ -46,10 +46,10 @@ def get_components(self) -> Dict[str, autoPyTorchComponent]: return components def get_available_components( - self, - dataset_properties: Optional[Dict[str, str]] = None, - include: List[str] = None, - exclude: List[str] = None, + self, + dataset_properties: Optional[Dict[str, str]] = None, + include: List[str] = None, + exclude: List[str] = None, ) -> Dict[str, autoPyTorchComponent]: """Filters out components based on user provided include/exclude directives, as well as the dataset properties @@ -102,11 +102,11 @@ def get_available_components( return components_dict def get_hyperparameter_search_space( - self, - dataset_properties: Optional[Dict[str, str]] = None, - default: Optional[str] = None, - include: Optional[List[str]] = None, - exclude: Optional[List[str]] = None, + self, + dataset_properties: Optional[Dict[str, str]] = None, + default: Optional[str] = None, + include: Optional[List[str]] = None, + exclude: Optional[List[str]] = None, ) -> ConfigurationSpace: """Returns the configuration space of the current chosen components @@ -141,14 +141,25 @@ def get_hyperparameter_search_space( if default_ in initializers: default = default_ break - - initializer = CSH.CategoricalHyperparameter( - '__choice__', - list(initializers.keys()), - default_value=default - ) + updates = self._get_search_space_updates() + if '__choice__' in updates.keys(): + choice_hyperparameter = updates['__choice__'] + if not set(choice_hyperparameter.value_range).issubset(initializers): + raise ValueError("Expected given update for {} to have " + "choices in {} got {}".format(self.__class__.__name__, + initializers, + choice_hyperparameter.value_range)) + initializer = CSH.CategoricalHyperparameter('__choice__', + choice_hyperparameter.value_range, + default_value=choice_hyperparameter.default_value) + else: + initializer = CSH.CategoricalHyperparameter( + '__choice__', + list(initializers.keys()), + default_value=default + ) cs.add_hyperparameter(initializer) - for name in initializers: + for name in initializer.choices: updates = self._get_search_space_updates(prefix=name) config_space = initializers[name].get_hyperparameter_search_space(dataset_properties, # type:ignore **updates) diff --git a/autoPyTorch/pipeline/components/setup/network_initializer/base_network_initializer.py b/autoPyTorch/pipeline/components/setup/network_initializer/base_network_initializer.py index 062cdb50d..c35c812e7 100644 --- a/autoPyTorch/pipeline/components/setup/network_initializer/base_network_initializer.py +++ b/autoPyTorch/pipeline/components/setup/network_initializer/base_network_initializer.py @@ -1,5 +1,5 @@ from abc import abstractmethod -from typing import Any, Callable, Dict, Optional, Tuple +from typing import Any, Callable, Dict, Optional from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( diff --git a/autoPyTorch/pipeline/components/setup/optimizer/AdamOptimizer.py b/autoPyTorch/pipeline/components/setup/optimizer/AdamOptimizer.py index 09a417075..b1abbe7e6 100644 --- a/autoPyTorch/pipeline/components/setup/optimizer/AdamOptimizer.py +++ b/autoPyTorch/pipeline/components/setup/optimizer/AdamOptimizer.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple +from typing import Any, Dict, Optional from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( diff --git a/autoPyTorch/pipeline/components/setup/optimizer/AdamWOptimizer.py b/autoPyTorch/pipeline/components/setup/optimizer/AdamWOptimizer.py index a79e62f34..504b594b3 100644 --- a/autoPyTorch/pipeline/components/setup/optimizer/AdamWOptimizer.py +++ b/autoPyTorch/pipeline/components/setup/optimizer/AdamWOptimizer.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple +from typing import Any, Dict, Optional from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( diff --git a/autoPyTorch/pipeline/components/setup/optimizer/RMSpropOptimizer.py b/autoPyTorch/pipeline/components/setup/optimizer/RMSpropOptimizer.py index 59c2753a7..8e7f0020a 100644 --- a/autoPyTorch/pipeline/components/setup/optimizer/RMSpropOptimizer.py +++ b/autoPyTorch/pipeline/components/setup/optimizer/RMSpropOptimizer.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple +from typing import Any, Dict, Optional from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( diff --git a/autoPyTorch/pipeline/components/setup/optimizer/SGDOptimizer.py b/autoPyTorch/pipeline/components/setup/optimizer/SGDOptimizer.py index 0f0f64fa6..96356145b 100644 --- a/autoPyTorch/pipeline/components/setup/optimizer/SGDOptimizer.py +++ b/autoPyTorch/pipeline/components/setup/optimizer/SGDOptimizer.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple +from typing import Any, Dict, Optional from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( diff --git a/autoPyTorch/pipeline/components/setup/optimizer/base_optimizer_choice.py b/autoPyTorch/pipeline/components/setup/optimizer/base_optimizer_choice.py index 5196f0bb7..93f61e74b 100644 --- a/autoPyTorch/pipeline/components/setup/optimizer/base_optimizer_choice.py +++ b/autoPyTorch/pipeline/components/setup/optimizer/base_optimizer_choice.py @@ -143,14 +143,25 @@ def get_hyperparameter_search_space( if default_ in available_optimizer: default = default_ break - - optimizer = CSH.CategoricalHyperparameter( - '__choice__', - list(available_optimizer.keys()), - default_value=default - ) + updates = self._get_search_space_updates() + if '__choice__' in updates.keys(): + choice_hyperparameter = updates['__choice__'] + if not set(choice_hyperparameter.value_range).issubset(available_optimizer): + raise ValueError("Expected given update for {} to have " + "choices in {} got {}".format(self.__class__.__name__, + available_optimizer, + choice_hyperparameter.value_range)) + optimizer = CSH.CategoricalHyperparameter('__choice__', + choice_hyperparameter.value_range, + default_value=choice_hyperparameter.default_value) + else: + optimizer = CSH.CategoricalHyperparameter( + '__choice__', + list(available_optimizer.keys()), + default_value=default + ) cs.add_hyperparameter(optimizer) - for name in available_optimizer: + for name in optimizer.choices: updates = self._get_search_space_updates(prefix=name) config_space = available_optimizer[name].get_hyperparameter_search_space(dataset_properties, # type: ignore **updates) diff --git a/autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py b/autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py index 85557b2e2..afb43ea97 100644 --- a/autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py +++ b/autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple +from typing import Any, Dict, Optional from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( @@ -205,10 +205,7 @@ def check_requirements(self, X: Dict[str, Any], y: Any = None) -> None: # or from X, Y pairs if 'split_id' not in X: raise ValueError("To fit a data loader, expected fit dictionary to have split_id. " - "Currently X={}.".format( - X - ) - ) + "Currently X={}.".format(X)) if 'backend' not in X: raise ValueError("backend is needed to load the data from disk") diff --git a/autoPyTorch/pipeline/components/training/trainer/base_trainer_choice.py b/autoPyTorch/pipeline/components/training/trainer/base_trainer_choice.py index 564d199dd..248d8085b 100755 --- a/autoPyTorch/pipeline/components/training/trainer/base_trainer_choice.py +++ b/autoPyTorch/pipeline/components/training/trainer/base_trainer_choice.py @@ -137,14 +137,25 @@ def get_hyperparameter_search_space( if default_ in available_trainers: default = default_ break - - trainer = CategoricalHyperparameter( - '__choice__', - list(available_trainers.keys()), - default_value=default - ) + updates = self._get_search_space_updates() + if '__choice__' in updates.keys(): + choice_hyperparameter = updates['__choice__'] + if not set(choice_hyperparameter.value_range).issubset(available_trainers): + raise ValueError("Expected given update for {} to have " + "choices in {} got {}".format(self.__class__.__name__, + available_trainers, + choice_hyperparameter.value_range)) + trainer = CategoricalHyperparameter('__choice__', + choice_hyperparameter.value_range, + default_value=choice_hyperparameter.default_value) + else: + trainer = CategoricalHyperparameter( + '__choice__', + list(available_trainers.keys()), + default_value=default + ) cs.add_hyperparameter(trainer) - for name in available_trainers: + for name in trainer.choices: updates = self._get_search_space_updates(prefix=name) config_space = available_trainers[name].get_hyperparameter_search_space(dataset_properties, # type:ignore **updates) diff --git a/autoPyTorch/pipeline/tabular_classification.py b/autoPyTorch/pipeline/tabular_classification.py index 73dca2878..752d7c03a 100644 --- a/autoPyTorch/pipeline/tabular_classification.py +++ b/autoPyTorch/pipeline/tabular_classification.py @@ -196,9 +196,8 @@ def _get_hyperparameter_search_space(self, embeddings = cs.get_hyperparameter('network_embedding:__choice__').choices if 'LearnedEntityEmbedding' in embeddings: encoders = cs.get_hyperparameter('encoder:__choice__').choices - default = cs.get_hyperparameter('network_embedding:__choice__').default_value possible_default_embeddings = copy.copy(list(embeddings)) - del possible_default_embeddings[possible_default_embeddings.index(default)] + del possible_default_embeddings[possible_default_embeddings.index('LearnedEntityEmbedding')] for encoder in encoders: if encoder == 'OneHotEncoder': diff --git a/autoPyTorch/utils/common.py b/autoPyTorch/utils/common.py index 6e48929d2..e49e15050 100644 --- a/autoPyTorch/utils/common.py +++ b/autoPyTorch/utils/common.py @@ -3,11 +3,11 @@ from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( - Hyperparameter, CategoricalHyperparameter, Constant, + Hyperparameter, + UniformFloatHyperparameter, UniformIntegerHyperparameter, - UniformFloatHyperparameter ) import numpy as np @@ -204,11 +204,15 @@ def get_hyperparameter(hyperparameter: HyperparameterSearchSpace, if len(hyperparameter.value_range) == 0: raise ValueError(hyperparameter.hyperparameter + ': The range has to contain at least one element') if len(hyperparameter.value_range) == 1 and hyperparameter_type != CategoricalHyperparameter: - return Constant(hyperparameter.hyperparameter, int(hyperparameter.value_range[0]) - if isinstance(hyperparameter.value_range[0], bool) else hyperparameter.value_range[0]) + if isinstance(hyperparameter.value_range[0], bool): + return Constant(hyperparameter.hyperparameter, hyperparameter.value_range[0]) + else: + return Constant(hyperparameter.hyperparameter, int(hyperparameter.value_range[0])) if len(hyperparameter.value_range) == 2 and hyperparameter.value_range[0] == hyperparameter.value_range[1]: - return Constant(hyperparameter.hyperparameter, int(hyperparameter.value_range[0]) - if isinstance(hyperparameter.value_range[0], bool) else hyperparameter.value_range[0]) + if isinstance(hyperparameter.value_range[0], bool): + return Constant(hyperparameter.hyperparameter, hyperparameter.value_range[0]) + else: + return Constant(hyperparameter.hyperparameter, int(hyperparameter.value_range[0])) if hyperparameter_type == CategoricalHyperparameter: return CategoricalHyperparameter(hyperparameter.hyperparameter, choices=hyperparameter.value_range, diff --git a/autoPyTorch/utils/hyperparameter_search_space_update.py b/autoPyTorch/utils/hyperparameter_search_space_update.py index 4be49cb8e..9891f5f45 100644 --- a/autoPyTorch/utils/hyperparameter_search_space_update.py +++ b/autoPyTorch/utils/hyperparameter_search_space_update.py @@ -1,6 +1,5 @@ import ast import os -import re from typing import List, Optional, Sequence, Tuple, Union from sklearn.base import BaseEstimator diff --git a/test/test_pipeline/test_tabular_classification.py b/test/test_pipeline/test_tabular_classification.py index ef508dc7b..014fad5df 100644 --- a/test/test_pipeline/test_tabular_classification.py +++ b/test/test_pipeline/test_tabular_classification.py @@ -330,4 +330,27 @@ def test_set_range_search_space_updates(self, fit_dictionary_tabular): except AssertionError as e: # As we are setting num_layers to 1 for fully connected # head, units_layer does not exist in the configspace - assert 'fully_connected:units_layer' in e.args[0] + assert 'fully_connected:units_layer' in e.args[0], e.args[0] + + def test_set_choices_updates(self, fit_dictionary_tabular): + dataset_properties = {'numerical_columns': [1], 'categorical_columns': [2], + 'task_type': 'tabular_classification'} + config_dict = TabularClassificationPipeline(dataset_properties=dataset_properties). \ + get_hyperparameter_search_space()._hyperparameters + updates = HyperparameterSearchSpaceUpdates() + for i, (name, hyperparameter) in enumerate(config_dict.items()): + if '__choice__' not in name: + continue + name = name.split(':') + hyperparameter_name = ':'.join(name[1:]) + if name[0] == 'network_embedding' and hyperparameter_name == '__choice__': + value_range = ('NoEmbedding',) + default_value = 'NoEmbedding' + else: + value_range = (hyperparameter.choices[0],) + default_value = hyperparameter.choices[0] + updates.append(node_name=name[0], hyperparameter=hyperparameter_name, + value_range=value_range, default_value=default_value) + pipeline = TabularClassificationPipeline(dataset_properties=dataset_properties, + search_space_updates=updates) + self._assert_pipeline_search_space(pipeline, updates) From 6e9cab9a18cccdd27005502d62e8c65f5c95bcfc Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Tue, 23 Mar 2021 14:32:43 +0100 Subject: [PATCH 04/10] fixed mypy bug and bug in making constant float hyperparameters --- autoPyTorch/pipeline/base_pipeline.py | 14 +++++++------- autoPyTorch/pipeline/components/base_choice.py | 5 +++-- .../pipeline/components/base_component.py | 16 +++------------- .../feature_preprocessing/KernelPCA.py | 6 +++--- .../feature_preprocessing/Nystroem.py | 8 ++++---- .../feature_preprocessing/RandomKitchenSinks.py | 6 +++--- .../feature_preprocessing/TruncatedSVD.py | 6 +++--- .../network_backbone/DenseNetImageBackone.py | 4 ++-- .../setup/network_backbone/MLPBackbone.py | 12 ++++++------ .../setup/network_backbone/ResNetBackbone.py | 2 +- .../network_backbone/ShapedResNetBackbone.py | 4 ++-- .../setup/network_backbone/TCNBackbone.py | 4 ++-- .../setup/network_head/fully_connected.py | 4 ++-- .../setup/network_head/fully_convolutional.py | 4 ++-- .../components/setup/optimizer/AdamWOptimizer.py | 2 +- autoPyTorch/utils/common.py | 14 ++++---------- 16 files changed, 48 insertions(+), 63 deletions(-) diff --git a/autoPyTorch/pipeline/base_pipeline.py b/autoPyTorch/pipeline/base_pipeline.py index 1bc94d3a5..792268487 100644 --- a/autoPyTorch/pipeline/base_pipeline.py +++ b/autoPyTorch/pipeline/base_pipeline.py @@ -417,18 +417,18 @@ def _check_search_space_updates(self, include: Optional[Dict[str, Any]], # the components in the value range of search space update # are in components of the choice module if split_hyperparameter[0] == '__choice__': - for component in update.value_range: + for choice in update.value_range: if include is not None and update.node_name in include.keys(): - if component not in include[update.node_name]: - raise ValueError("Not found {} in include".format(component)) + if choice not in include[update.node_name]: + raise ValueError("Not found {} in include".format(choice)) if exclude is not None and update.node_name in exclude.keys(): - if component in exclude[update.node_name]: - raise ValueError("Found {} in exclude".format(component)) - if component not in components.keys(): + if choice in exclude[update.node_name]: + raise ValueError("Found {} in exclude".format(choice)) + if choice not in components.keys(): raise ValueError("Unknown hyperparameter for choice {}. " "Expected update hyperparameter " "to be in {} got {}".format(node.__class__.__name__, - components.keys(), component)) + components.keys(), choice)) # check if the component whose hyperparameter # needs to be updated is in components of the # choice module diff --git a/autoPyTorch/pipeline/components/base_choice.py b/autoPyTorch/pipeline/components/base_choice.py index 0aab76516..537e2e963 100644 --- a/autoPyTorch/pipeline/components/base_choice.py +++ b/autoPyTorch/pipeline/components/base_choice.py @@ -252,7 +252,7 @@ def _check_dataset_properties(self, dataset_properties: Dict[str, Any]) -> None: def _apply_search_space_update(self, hyperparameter_search_space_update: HyperparameterSearchSpaceUpdate) -> None: """Allows the user to update a hyperparameter - Arguments: + Args: name {string} -- name of hyperparameter new_value_range {List[?] -- value range can be either lower, upper or a list of possible conditionals log {bool} -- is hyperparameter logscale @@ -263,7 +263,7 @@ def _apply_search_space_update(self, hyperparameter_search_space_update: Hyperpa def _get_search_space_updates(self, prefix: Optional[str] = None) -> Dict[str, HyperparameterSearchSpace]: """Get the search space updates with the given prefix - Keyword Arguments: + Args: prefix {str} -- Only return search space updates with given prefix (default: {None}) Returns: @@ -280,5 +280,6 @@ def _get_search_space_updates(self, prefix: Optional[str] = None) -> Dict[str, H if RETURN_ALL: result[key] = self._cs_updates[key].get_search_space() elif re.search(f'^{prefix}', key) is not None: + assert isinstance(prefix, str) result[key[len(prefix) + 1:]] = self._cs_updates[key].get_search_space(remove_prefix=prefix) return result diff --git a/autoPyTorch/pipeline/components/base_component.py b/autoPyTorch/pipeline/components/base_component.py index 981392671..09d981342 100644 --- a/autoPyTorch/pipeline/components/base_component.py +++ b/autoPyTorch/pipeline/components/base_component.py @@ -1,7 +1,6 @@ import importlib import inspect import pkgutil -import re import sys import warnings from collections import OrderedDict @@ -265,25 +264,16 @@ def _apply_search_space_update(self, hyperparameter_search_space_update: Hyperpa self._cs_updates[hyperparameter_search_space_update.hyperparameter] = hyperparameter_search_space_update - def _get_search_space_updates(self, prefix: Optional[str] = None) -> Dict[str, HyperparameterSearchSpace]: - """Get the search space updates with the given prefix - - Keyword Arguments: - prefix {str} -- Only return search space updates with given prefix (default: {None}) + def _get_search_space_updates(self) -> Dict[str, HyperparameterSearchSpace]: + """Get the search space updates Returns: dict -- Mapping of search space updates. Keys don't contain the prefix. """ - RETURN_ALL = False - if prefix is None: - RETURN_ALL = True result: Dict[str, HyperparameterSearchSpace] = dict() # iterate over all search space updates of this node and keep the ones that have the given prefix for key in self._cs_updates.keys(): - if RETURN_ALL: - result[key] = self._cs_updates[key].get_search_space() - elif re.search(f'^{prefix}', key) is not None: - result[key[len(prefix) + 1:]] = self._cs_updates[key].get_search_space(remove_prefix=prefix) + result[key] = self._cs_updates[key].get_search_space() return result diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/KernelPCA.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/KernelPCA.py index ee471a71f..ef8bcc9ef 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/KernelPCA.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/KernelPCA.py @@ -79,9 +79,9 @@ def get_hyperparameter_search_space( else: log = n_components.log n_components = HyperparameterSearchSpace(hyperparameter='n_components', - value_range=(floor(n_components.value_range[0] * n_features), - ceil(n_components.value_range[1] * n_features)), - default_value=ceil(n_components.default_value * n_features), + value_range=(floor(int(n_components.value_range[0]) * n_features), + ceil(int(n_components.value_range[1]) * n_features)), + default_value=ceil(int(n_components.default_value) * n_features), log=log) else: n_components = HyperparameterSearchSpace(hyperparameter='n_components', diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/Nystroem.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/Nystroem.py index e4e007e61..2a4c2cae8 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/Nystroem.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/Nystroem.py @@ -75,11 +75,11 @@ def get_hyperparameter_search_space( if n_features == 1: log = False else: - log = n_components[2] + log = n_components.log n_components = HyperparameterSearchSpace(hyperparameter='n_components', - value_range=(floor(n_components.value_range[0] * n_features), - ceil(n_components.value_range[1] * n_features)), - default_value=ceil(n_components.default_value * n_features), + value_range=(floor(int(n_components.value_range[0]) * n_features), + ceil(int(n_components.value_range[1]) * n_features)), + default_value=ceil(int(n_components.default_value) * n_features), log=log) else: n_components = HyperparameterSearchSpace(hyperparameter='n_components', diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/RandomKitchenSinks.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/RandomKitchenSinks.py index b2643e768..32e9aade0 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/RandomKitchenSinks.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/RandomKitchenSinks.py @@ -54,9 +54,9 @@ def get_hyperparameter_search_space( else: log = n_components.log n_components = HyperparameterSearchSpace(hyperparameter='n_components', - value_range=(floor(n_components.value_range[0] * n_features), - ceil(n_components.value_range[1] * n_features)), - default_value=ceil(n_components.default_value * n_features), + value_range=(floor(int(n_components.value_range[0]) * n_features), + ceil(int(n_components.value_range[1]) * n_features)), + default_value=ceil(int(n_components.default_value) * n_features), log=log) else: n_components = HyperparameterSearchSpace(hyperparameter='n_components', diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/TruncatedSVD.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/TruncatedSVD.py index 7fdccced2..ff2cb792d 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/TruncatedSVD.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/TruncatedSVD.py @@ -49,9 +49,9 @@ def get_hyperparameter_search_space( if dataset_properties is not None: n_features = len(dataset_properties['numerical_columns']) target_dim = HyperparameterSearchSpace(hyperparameter=target_dim.hyperparameter, - value_range=(floor(target_dim.value_range[0] * n_features), - floor(target_dim.value_range[1] * n_features)), - default_value=floor(target_dim.default_value * n_features), + value_range=(floor(int(target_dim.value_range[0]) * n_features), + floor(int(target_dim.value_range[1]) * n_features)), + default_value=floor(int(target_dim.default_value) * n_features), log=target_dim.log) else: target_dim = HyperparameterSearchSpace(hyperparameter=target_dim.hyperparameter, diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/DenseNetImageBackone.py b/autoPyTorch/pipeline/components/setup/network_backbone/DenseNetImageBackone.py index ea20b1727..23eddaf65 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/DenseNetImageBackone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/DenseNetImageBackone.py @@ -198,7 +198,7 @@ def get_hyperparameter_search_space( cs.add_hyperparameters([use_dropout, dropout]) cs.add_condition(CS.EqualsCondition(dropout, use_dropout, True)) - for i in range(1, max_num_blocks + 1): + for i in range(1, int(max_num_blocks) + 1): layer_search_space = HyperparameterSearchSpace(hyperparameter='layer_in_block_%d' % i, value_range=num_layers.value_range, @@ -206,7 +206,7 @@ def get_hyperparameter_search_space( log=num_layers.log) layer_hp = get_hyperparameter(layer_search_space, UniformIntegerHyperparameter) - if i > min_num_blocks: + if i > int(min_num_blocks): cs.add_condition(CS.GreaterThanCondition(layer_hp, blocks_hp, i - 1)) return cs diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/MLPBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/MLPBackbone.py index ade48155e..930b5acfe 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/MLPBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/MLPBackbone.py @@ -107,7 +107,7 @@ def get_hyperparameter_search_space( use_dropout = get_hyperparameter(use_dropout, CategoricalHyperparameter) cs.add_hyperparameters([num_groups, use_dropout]) - for i in range(1, max_mlp_layers + 1): + for i in range(1, int(max_mlp_layers) + 1): n_units_search_space = HyperparameterSearchSpace(hyperparameter='num_units_%d' % i, value_range=num_units.value_range, default_value=num_units.default_value, @@ -115,7 +115,7 @@ def get_hyperparameter_search_space( n_units_hp = get_hyperparameter(n_units_search_space, UniformIntegerHyperparameter) cs.add_hyperparameter(n_units_hp) - if i > min_mlp_layers: + if i > int(min_mlp_layers): # The units of layer i should only exist # if there are at least i layers cs.add_condition( @@ -124,15 +124,15 @@ def get_hyperparameter_search_space( ) ) dropout_search_space = HyperparameterSearchSpace(hyperparameter='dropout_%d' % i, - value_range=num_units.value_range, - default_value=num_units.default_value, - log=num_units.log) + value_range=dropout.value_range, + default_value=dropout.default_value, + log=dropout.log) dropout_hp = get_hyperparameter(dropout_search_space, UniformFloatHyperparameter) cs.add_hyperparameter(dropout_hp) dropout_condition_1 = CS.EqualsCondition(dropout_hp, use_dropout, True) - if i > min_mlp_layers: + if i > int(min_mlp_layers): dropout_condition_2 = CS.GreaterThanCondition(dropout_hp, num_groups, i - 1) cs.add_condition(CS.AndConjunction(dropout_condition_1, dropout_condition_2)) else: diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py index d57d783a4..6391baa6a 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py @@ -156,7 +156,7 @@ def get_hyperparameter_search_space( # It is the upper bound of the nr of groups, # since the configuration will actually be sampled. - for i in range(0, max_num_groups + 1): + for i in range(0, int(max_num_groups) + 1): n_units_search_space = HyperparameterSearchSpace(hyperparameter='num_units_%d' % i, value_range=num_units.value_range, diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py index 2988d4c6b..c8ffa1b4e 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py @@ -81,8 +81,8 @@ def get_properties(dataset_properties: Optional[Dict[str, Any]] = None) -> Dict[ } @staticmethod - def get_hyperparameter_search_space( - dataset_properties: Optional[Dict] = None, # type: ignore[override] + def get_hyperparameter_search_space( # type: ignore[override] + dataset_properties: Optional[Dict] = None, resnet_shape: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="resnet_shape", value_range=('funnel', 'long_funnel', 'diamond', 'hexagon', diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/TCNBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/TCNBackbone.py index 9e9ad5f31..5a20d945a 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/TCNBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/TCNBackbone.py @@ -156,14 +156,14 @@ def get_hyperparameter_search_space( cs.add_hyperparameter(dropout_hp) cs.add_condition(CS.EqualsCondition(dropout_hp, use_dropout_hp, True)) - for i in range(0, max_num_blocks): + for i in range(0, int(max_num_blocks)): num_filter_search_space = HyperparameterSearchSpace(f"num_filter_{i}", value_range=num_filters.value_range, default_value=num_filters.default_value, log=num_filters.log) num_filters_hp = get_hyperparameter(num_filter_search_space, UniformIntegerHyperparameter) cs.add_hyperparameter(num_filters_hp) - if i >= min_num_blocks: + if i >= int(min_num_blocks): cs.add_condition(CS.GreaterThanCondition( num_filters_hp, num_blocks_hp, i)) diff --git a/autoPyTorch/pipeline/components/setup/network_head/fully_connected.py b/autoPyTorch/pipeline/components/setup/network_head/fully_connected.py index ebfabdb92..93f653b37 100644 --- a/autoPyTorch/pipeline/components/setup/network_head/fully_connected.py +++ b/autoPyTorch/pipeline/components/setup/network_head/fully_connected.py @@ -65,7 +65,7 @@ def get_hyperparameter_search_space( cs.add_hyperparameters([num_layers_hp, activation_hp]) cs.add_condition(CS.GreaterThanCondition(activation_hp, num_layers_hp, 1)) - for i in range(1, max_num_layers): + for i in range(1, int(max_num_layers)): num_units_search_space = HyperparameterSearchSpace(hyperparameter=f"units_layer_{i}", value_range=units_layer.value_range, default_value=units_layer.default_value, @@ -73,7 +73,7 @@ def get_hyperparameter_search_space( num_units_hp = get_hyperparameter(num_units_search_space, UniformIntegerHyperparameter) cs.add_hyperparameter(num_units_hp) - if i >= min_num_layers: + if i >= int(min_num_layers): cs.add_condition(CS.GreaterThanCondition(num_units_hp, num_layers_hp, i)) return cs diff --git a/autoPyTorch/pipeline/components/setup/network_head/fully_convolutional.py b/autoPyTorch/pipeline/components/setup/network_head/fully_convolutional.py index 9388dd774..e291442ea 100644 --- a/autoPyTorch/pipeline/components/setup/network_head/fully_convolutional.py +++ b/autoPyTorch/pipeline/components/setup/network_head/fully_convolutional.py @@ -98,14 +98,14 @@ def get_hyperparameter_search_space( cs.add_hyperparameters([num_layers_hp, activation_hp]) cs.add_condition(CS.GreaterThanCondition(activation_hp, num_layers_hp, 1)) - for i in range(1, max_num_layers): + for i in range(1, int(max_num_layers)): num_filters_search_space = HyperparameterSearchSpace(f"layer_{i}_filters", value_range=num_filters.value_range, default_value=num_filters.default_value, log=num_filters.log) num_filters_hp = get_hyperparameter(num_filters_search_space, UniformIntegerHyperparameter) cs.add_hyperparameter(num_filters_hp) - if i >= min_num_layers: + if i >= int(min_num_layers): cs.add_condition(CS.GreaterThanCondition(num_filters_hp, num_layers_hp, i)) return cs diff --git a/autoPyTorch/pipeline/components/setup/optimizer/AdamWOptimizer.py b/autoPyTorch/pipeline/components/setup/optimizer/AdamWOptimizer.py index 504b594b3..658b378c8 100644 --- a/autoPyTorch/pipeline/components/setup/optimizer/AdamWOptimizer.py +++ b/autoPyTorch/pipeline/components/setup/optimizer/AdamWOptimizer.py @@ -88,7 +88,7 @@ def get_hyperparameter_search_space( weight_decay: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="weight_decay", value_range=(0.0, 0.1), default_value=0.0), - ) -> ConfigurationSpace(): + ) -> ConfigurationSpace: cs = ConfigurationSpace() # The learning rate for the model diff --git a/autoPyTorch/utils/common.py b/autoPyTorch/utils/common.py index e49e15050..a1227d560 100644 --- a/autoPyTorch/utils/common.py +++ b/autoPyTorch/utils/common.py @@ -204,15 +204,9 @@ def get_hyperparameter(hyperparameter: HyperparameterSearchSpace, if len(hyperparameter.value_range) == 0: raise ValueError(hyperparameter.hyperparameter + ': The range has to contain at least one element') if len(hyperparameter.value_range) == 1 and hyperparameter_type != CategoricalHyperparameter: - if isinstance(hyperparameter.value_range[0], bool): - return Constant(hyperparameter.hyperparameter, hyperparameter.value_range[0]) - else: - return Constant(hyperparameter.hyperparameter, int(hyperparameter.value_range[0])) + return Constant(hyperparameter.hyperparameter, hyperparameter.value_range[0]) if len(hyperparameter.value_range) == 2 and hyperparameter.value_range[0] == hyperparameter.value_range[1]: - if isinstance(hyperparameter.value_range[0], bool): - return Constant(hyperparameter.hyperparameter, hyperparameter.value_range[0]) - else: - return Constant(hyperparameter.hyperparameter, int(hyperparameter.value_range[0])) + return Constant(hyperparameter.hyperparameter, hyperparameter.value_range[0]) if hyperparameter_type == CategoricalHyperparameter: return CategoricalHyperparameter(hyperparameter.hyperparameter, choices=hyperparameter.value_range, @@ -240,7 +234,7 @@ def get_hyperparameter(hyperparameter: HyperparameterSearchSpace, def add_hyperparameter(cs: ConfigurationSpace, hyperparameter: HyperparameterSearchSpace, - hyperparameter_type: Type[Hyperparameter]): + hyperparameter_type: Type[Hyperparameter]) -> None: """ Adds the given hyperparameter to the given configuration space @@ -253,6 +247,6 @@ def add_hyperparameter(cs: ConfigurationSpace, type of the hyperparameter Returns: - + None """ return cs.add_hyperparameter(get_hyperparameter(hyperparameter, hyperparameter_type)) From b651a8904ce53624b12a4b52716b3d94da39f2a1 Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Tue, 23 Mar 2021 14:33:45 +0100 Subject: [PATCH 05/10] Add test for fitting pipeline with constant updates --- test/conftest.py | 4 +- .../components/setup/test_setup_networks.py | 2 - .../test_tabular_classification.py | 58 +++++++++++++++++++ 3 files changed, 61 insertions(+), 3 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 04077ca08..06f18e0f9 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -259,7 +259,9 @@ def get_tabular_data(task): y = y.iloc[0:200] y = (y - y.mean()) / y.std() validator = TabularInputValidator(is_classification=False).fit(X.copy(), y.copy()) - + elif task == 'iris': + X, y = fetch_openml("iris", return_X_y=True, as_frame=True) + validator = TabularInputValidator(is_classification=True).fit(X.copy(), y.copy()) else: raise ValueError("Unsupported task {}".format(task)) diff --git a/test/test_pipeline/components/setup/test_setup_networks.py b/test/test_pipeline/components/setup/test_setup_networks.py index f6f3decb0..6826d7ef2 100644 --- a/test/test_pipeline/components/setup/test_setup_networks.py +++ b/test/test_pipeline/components/setup/test_setup_networks.py @@ -47,8 +47,6 @@ def test_pipeline_fit(self, fit_dictionary_tabular, embedding, backbone, head): assert backbone == config.get('network_backbone:__choice__', None) assert head == config.get('network_head:__choice__', None) pipeline.set_hyperparameters(config) - # Need more epochs to make sure validation performance is met - fit_dictionary_tabular['epochs'] = 100 # Early stop to the best configuration seen fit_dictionary_tabular['early_stopping'] = 50 diff --git a/test/test_pipeline/test_tabular_classification.py b/test/test_pipeline/test_tabular_classification.py index 014fad5df..3925c9390 100644 --- a/test/test_pipeline/test_tabular_classification.py +++ b/test/test_pipeline/test_tabular_classification.py @@ -354,3 +354,61 @@ def test_set_choices_updates(self, fit_dictionary_tabular): pipeline = TabularClassificationPipeline(dataset_properties=dataset_properties, search_space_updates=updates) self._assert_pipeline_search_space(pipeline, updates) + + +@pytest.mark.parametrize("fit_dictionary_tabular", ['iris'], indirect=True) +def test_constant_pipeline_iris(fit_dictionary_tabular): + search_space_updates = HyperparameterSearchSpaceUpdates() + search_space_updates.append(node_name='network_backbone', + hyperparameter='__choice__', + value_range=['MLPBackbone'], + default_value='MLPBackbone') + search_space_updates.append(node_name='network_backbone', + hyperparameter='MLPBackbone:num_groups', + value_range=[1, 1], + default_value=1) + search_space_updates.append(node_name='network_backbone', + hyperparameter='MLPBackbone:num_units', + value_range=[100], + default_value=100) + search_space_updates.append(node_name='trainer', + hyperparameter='__choice__', + value_range=['StandardTrainer'], + default_value='StandardTrainer') + search_space_updates.append(node_name='lr_scheduler', + hyperparameter='__choice__', + value_range=['NoScheduler'], + default_value='NoScheduler') + search_space_updates.append(node_name='optimizer', + hyperparameter='__choice__', + value_range=['AdamOptimizer'], + default_value='AdamOptimizer') + search_space_updates.append(node_name='optimizer', + hyperparameter='AdamOptimizer:lr', + value_range=[1e-2], + default_value=1e-2) + pipeline = TabularClassificationPipeline(dataset_properties=fit_dictionary_tabular['dataset_properties'], + search_space_updates=search_space_updates) + + try: + pipeline.fit(fit_dictionary_tabular) + except Exception as e: + pytest.fail(f"Failed due to {e}") + + # To make sure we fitted the model, there should be a + # run summary object with accuracy + run_summary = pipeline.named_steps['trainer'].run_summary + assert run_summary is not None + + # Make sure that performance was properly captured + assert run_summary.performance_tracker['train_loss'][1] > 0 + assert run_summary.total_parameter_count > 0 + assert 'accuracy' in run_summary.performance_tracker['train_metrics'][1] + + # Make sure default pipeline achieves a good score for dummy datasets + epoch2loss = run_summary.performance_tracker['val_loss'] + best_loss = min(list(epoch2loss.values())) + epoch_where_best = list(epoch2loss.keys())[list(epoch2loss.values()).index(best_loss)] + score = run_summary.performance_tracker['val_metrics'][epoch_where_best]['accuracy'] + + assert score >= 0.9, run_summary.performance_tracker['val_metrics'] \ No newline at end of file From 2b23b640597f313135ea7116dcbe5b274bc48047 Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Tue, 23 Mar 2021 14:40:48 +0100 Subject: [PATCH 06/10] fix flake --- test/test_pipeline/test_tabular_classification.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_pipeline/test_tabular_classification.py b/test/test_pipeline/test_tabular_classification.py index 3925c9390..b752e52da 100644 --- a/test/test_pipeline/test_tabular_classification.py +++ b/test/test_pipeline/test_tabular_classification.py @@ -411,4 +411,4 @@ def test_constant_pipeline_iris(fit_dictionary_tabular): epoch_where_best = list(epoch2loss.keys())[list(epoch2loss.values()).index(best_loss)] score = run_summary.performance_tracker['val_metrics'][epoch_where_best]['accuracy'] - assert score >= 0.9, run_summary.performance_tracker['val_metrics'] \ No newline at end of file + assert score >= 0.9, run_summary.performance_tracker['val_metrics'] From 78f9ea4c6c02f9715296aba779190f5a6662b8c9 Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Tue, 23 Mar 2021 16:29:33 +0100 Subject: [PATCH 07/10] bug in int for feature preprocessors and minor bugs in hyperparameter search space fixed --- .../feature_preprocessing/KernelPCA.py | 7 ++-- .../feature_preprocessing/Nystroem.py | 9 ++-- .../RandomKitchenSinks.py | 7 ++-- .../feature_preprocessing/TruncatedSVD.py | 6 +-- .../setup/augmentation/image/GaussianBlur.py | 2 +- .../network_backbone/ConvNetImageBackbone.py | 42 +++++++++---------- ...ageBackone.py => DenseNetImageBackbone.py} | 3 +- .../setup/network_backbone/TCNBackbone.py | 2 +- 8 files changed, 41 insertions(+), 37 deletions(-) rename autoPyTorch/pipeline/components/setup/network_backbone/{DenseNetImageBackone.py => DenseNetImageBackbone.py} (99%) diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/KernelPCA.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/KernelPCA.py index ef8bcc9ef..2a4737c4d 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/KernelPCA.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/KernelPCA.py @@ -79,9 +79,10 @@ def get_hyperparameter_search_space( else: log = n_components.log n_components = HyperparameterSearchSpace(hyperparameter='n_components', - value_range=(floor(int(n_components.value_range[0]) * n_features), - ceil(int(n_components.value_range[1]) * n_features)), - default_value=ceil(int(n_components.default_value) * n_features), + value_range=( + floor(float(n_components.value_range[0]) * n_features), + ceil(float(n_components.value_range[1]) * n_features)), + default_value=ceil(float(n_components.default_value) * n_features), log=log) else: n_components = HyperparameterSearchSpace(hyperparameter='n_components', diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/Nystroem.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/Nystroem.py index 2a4c2cae8..0a8f6c63d 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/Nystroem.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/Nystroem.py @@ -14,7 +14,7 @@ import sklearn.kernel_approximation from sklearn.base import BaseEstimator -from autoPyTorch.pipeline.components.preprocessing.tabular_preprocessing.feature_preprocessing.\ +from autoPyTorch.pipeline.components.preprocessing.tabular_preprocessing.feature_preprocessing. \ base_feature_preprocessor import autoPyTorchFeaturePreprocessingComponent from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter @@ -77,9 +77,10 @@ def get_hyperparameter_search_space( else: log = n_components.log n_components = HyperparameterSearchSpace(hyperparameter='n_components', - value_range=(floor(int(n_components.value_range[0]) * n_features), - ceil(int(n_components.value_range[1]) * n_features)), - default_value=ceil(int(n_components.default_value) * n_features), + value_range=( + floor(float(n_components.value_range[0]) * n_features), + ceil(float(n_components.value_range[1]) * n_features)), + default_value=ceil(float(n_components.default_value) * n_features), log=log) else: n_components = HyperparameterSearchSpace(hyperparameter='n_components', diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/RandomKitchenSinks.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/RandomKitchenSinks.py index 32e9aade0..a10695929 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/RandomKitchenSinks.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/RandomKitchenSinks.py @@ -54,9 +54,10 @@ def get_hyperparameter_search_space( else: log = n_components.log n_components = HyperparameterSearchSpace(hyperparameter='n_components', - value_range=(floor(int(n_components.value_range[0]) * n_features), - ceil(int(n_components.value_range[1]) * n_features)), - default_value=ceil(int(n_components.default_value) * n_features), + value_range=( + floor(float(n_components.value_range[0]) * n_features), + ceil(float(n_components.value_range[1]) * n_features)), + default_value=ceil(float(n_components.default_value) * n_features), log=log) else: n_components = HyperparameterSearchSpace(hyperparameter='n_components', diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/TruncatedSVD.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/TruncatedSVD.py index ff2cb792d..fdf6a751a 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/TruncatedSVD.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/TruncatedSVD.py @@ -49,9 +49,9 @@ def get_hyperparameter_search_space( if dataset_properties is not None: n_features = len(dataset_properties['numerical_columns']) target_dim = HyperparameterSearchSpace(hyperparameter=target_dim.hyperparameter, - value_range=(floor(int(target_dim.value_range[0]) * n_features), - floor(int(target_dim.value_range[1]) * n_features)), - default_value=floor(int(target_dim.default_value) * n_features), + value_range=(floor(float(target_dim.value_range[0]) * n_features), + floor(float(target_dim.value_range[1]) * n_features)), + default_value=floor(float(target_dim.default_value) * n_features), log=target_dim.log) else: target_dim = HyperparameterSearchSpace(hyperparameter=target_dim.hyperparameter, diff --git a/autoPyTorch/pipeline/components/setup/augmentation/image/GaussianBlur.py b/autoPyTorch/pipeline/components/setup/augmentation/image/GaussianBlur.py index 962d959d8..8fca40999 100644 --- a/autoPyTorch/pipeline/components/setup/augmentation/image/GaussianBlur.py +++ b/autoPyTorch/pipeline/components/setup/augmentation/image/GaussianBlur.py @@ -40,7 +40,7 @@ def get_hyperparameter_search_space( value_range=(0, 3), default_value=0, ), - sigma_offset: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="sigma_min", + sigma_offset: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="sigma_offset", value_range=(0.0, 3.0), default_value=0.5, ), diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/ConvNetImageBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/ConvNetImageBackbone.py index 767675216..162e9f4e6 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/ConvNetImageBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/ConvNetImageBackbone.py @@ -74,26 +74,26 @@ def get_hyperparameter_search_space( value_range=(2, 8), default_value=4, ), - num_init_filters: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='num_init_filters', - value_range=(16, 64), - default_value=32, - ), + conv_init_filters: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='conv_init_filters', + value_range=(16, 64), + default_value=32, + ), activation: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='activation', value_range=tuple(_activations.keys()), default_value=list(_activations.keys())[0], ), - kernel_size: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='kernel_size', - value_range=(3, 5), - default_value=3, - ), - stride: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='stride', - value_range=(1, 3), - default_value=1, - ), - padding: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='padding', - value_range=(2, 3), - default_value=2, - ), + conv_kernel_size: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='conv_kernel_size', + value_range=(3, 5), + default_value=3, + ), + conv_kernel_stride: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='conv_kernel_stride', + value_range=(1, 3), + default_value=1, + ), + conv_kernel_padding: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='conv_kernel_padding', + value_range=(2, 3), + default_value=2, + ), pool_size: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='pool_size', value_range=(2, 3), default_value=2, @@ -101,12 +101,12 @@ def get_hyperparameter_search_space( ) -> ConfigurationSpace: cs = CS.ConfigurationSpace() - add_hyperparameter(cs, num_init_filters, UniformIntegerHyperparameter) - add_hyperparameter(cs, num_init_filters, UniformIntegerHyperparameter) + add_hyperparameter(cs, num_layers, UniformIntegerHyperparameter) + add_hyperparameter(cs, conv_init_filters, UniformIntegerHyperparameter) add_hyperparameter(cs, activation, CategoricalHyperparameter) - add_hyperparameter(cs, kernel_size, UniformIntegerHyperparameter) - add_hyperparameter(cs, stride, UniformIntegerHyperparameter) - add_hyperparameter(cs, padding, UniformIntegerHyperparameter) + add_hyperparameter(cs, conv_kernel_size, UniformIntegerHyperparameter) + add_hyperparameter(cs, conv_kernel_stride, UniformIntegerHyperparameter) + add_hyperparameter(cs, conv_kernel_padding, UniformIntegerHyperparameter) add_hyperparameter(cs, pool_size, UniformIntegerHyperparameter) return cs diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/DenseNetImageBackone.py b/autoPyTorch/pipeline/components/setup/network_backbone/DenseNetImageBackbone.py similarity index 99% rename from autoPyTorch/pipeline/components/setup/network_backbone/DenseNetImageBackone.py rename to autoPyTorch/pipeline/components/setup/network_backbone/DenseNetImageBackbone.py index 23eddaf65..f32cd1640 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/DenseNetImageBackone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/DenseNetImageBackbone.py @@ -93,7 +93,7 @@ def build_backbone(self, input_shape: Tuple[int, ...]) -> nn.Module: channels, iw, ih = input_shape growth_rate = self.config['growth_rate'] - block_config = [self.config['layer_in_block_%d' % (i + 1)] for i in range(self.config['blocks'])] + block_config = [self.config['layer_in_block_%d' % (i + 1)] for i in range(self.config['num_blocks'])] num_init_features = 2 * growth_rate bn_size = 4 drop_rate = self.config['dropout'] if self.config['use_dropout'] else 0 @@ -206,6 +206,7 @@ def get_hyperparameter_search_space( log=num_layers.log) layer_hp = get_hyperparameter(layer_search_space, UniformIntegerHyperparameter) + cs.add_hyperparameter(layer_hp) if i > int(min_num_blocks): cs.add_condition(CS.GreaterThanCondition(layer_hp, blocks_hp, i - 1)) diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/TCNBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/TCNBackbone.py index 5a20d945a..b7129c226 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/TCNBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/TCNBackbone.py @@ -157,7 +157,7 @@ def get_hyperparameter_search_space( cs.add_condition(CS.EqualsCondition(dropout_hp, use_dropout_hp, True)) for i in range(0, int(max_num_blocks)): - num_filter_search_space = HyperparameterSearchSpace(f"num_filter_{i}", + num_filter_search_space = HyperparameterSearchSpace(f"num_filters_{i}", value_range=num_filters.value_range, default_value=num_filters.default_value, log=num_filters.log) From ad43a947a5447c75b4ccd6bd44335fb96aa226d8 Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Tue, 23 Mar 2021 17:18:36 +0100 Subject: [PATCH 08/10] Forgot to add a file --- test/test_pipeline/components/setup/test_setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_pipeline/components/setup/test_setup.py b/test/test_pipeline/components/setup/test_setup.py index 9349c4ac8..aae1c8ff4 100644 --- a/test/test_pipeline/components/setup/test_setup.py +++ b/test/test_pipeline/components/setup/test_setup.py @@ -309,7 +309,7 @@ def test_dummy_forward_backward_pass(self, task_type_input_shape): value_range=[1, 3], default_value=2) updates.append(node_name='network_backbone', - hyperparameter='ConvNetImageBackbone:num_init_filters', + hyperparameter='ConvNetImageBackbone:conv_init_filters', value_range=[8, 16], default_value=8) updates.append(node_name='network_backbone', From 604fedc28963337c7877cb546339af560c8a391b Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Wed, 24 Mar 2021 16:50:48 +0100 Subject: [PATCH 09/10] Addressed comments, better documentation and better tests for search space updates --- .../pipeline/components/base_choice.py | 20 ++--- .../RandomKitchenSinks.py | 2 +- .../scaling/base_scaler_choice.py | 7 +- .../base_network_embedding_choice.py | 6 +- .../pipeline/tabular_classification.py | 3 +- autoPyTorch/utils/common.py | 19 ++++- .../test_tabular_classification.py | 34 +++++++- ...test_hyperparameter_search_space_update.py | 83 +++++++++++++++++++ 8 files changed, 146 insertions(+), 28 deletions(-) create mode 100644 test/test_utils/test_hyperparameter_search_space_update.py diff --git a/autoPyTorch/pipeline/components/base_choice.py b/autoPyTorch/pipeline/components/base_choice.py index 537e2e963..53e94e9f3 100644 --- a/autoPyTorch/pipeline/components/base_choice.py +++ b/autoPyTorch/pipeline/components/base_choice.py @@ -250,12 +250,12 @@ def _check_dataset_properties(self, dataset_properties: Dict[str, Any]) -> None: assert isinstance(dataset_properties, dict), "dataset_properties must be a dictionary" def _apply_search_space_update(self, hyperparameter_search_space_update: HyperparameterSearchSpaceUpdate) -> None: - """Allows the user to update a hyperparameter + """ + Applies search space update to the class Args: - name {string} -- name of hyperparameter - new_value_range {List[?] -- value range can be either lower, upper or a list of possible conditionals - log {bool} -- is hyperparameter logscale + hyperparameter_search_space_update (HyperparameterSearchSpaceUpdate): + Search Space update for the current autoPyTorchChoice module """ self._cs_updates[hyperparameter_search_space_update.hyperparameter] = hyperparameter_search_space_update @@ -264,22 +264,20 @@ def _get_search_space_updates(self, prefix: Optional[str] = None) -> Dict[str, H """Get the search space updates with the given prefix Args: - prefix {str} -- Only return search space updates with given prefix (default: {None}) + prefix (str): + Only return search space updates with given prefix (default: {None}) Returns: - dict -- Mapping of search space updates. Keys don't contain the prefix. + Dict[str, HyperparameterSearchSpace]: + Mapping of search space updates. Keys don't contain the prefix. """ - RETURN_ALL = False - if prefix is None: - RETURN_ALL = True result: Dict[str, HyperparameterSearchSpace] = dict() # iterate over all search space updates of this node and keep the ones that have the given prefix for key in self._cs_updates.keys(): - if RETURN_ALL: + if prefix is None: result[key] = self._cs_updates[key].get_search_space() elif re.search(f'^{prefix}', key) is not None: - assert isinstance(prefix, str) result[key[len(prefix) + 1:]] = self._cs_updates[key].get_search_space(remove_prefix=prefix) return result diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/RandomKitchenSinks.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/RandomKitchenSinks.py index a10695929..9dbf26cbc 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/RandomKitchenSinks.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/RandomKitchenSinks.py @@ -42,7 +42,7 @@ def get_hyperparameter_search_space( ), gamma: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='gamma', value_range=(3.0517578125e-05, 8), - default_value=0.01, + default_value=1.0, log=True), ) -> ConfigurationSpace: cs = ConfigurationSpace() diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/base_scaler_choice.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/base_scaler_choice.py index ec7416005..8e32c0cd5 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/base_scaler_choice.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/base_scaler_choice.py @@ -80,9 +80,10 @@ def get_hyperparameter_search_space(self, choice_hyperparameter.value_range)) if len(dataset_properties['numerical_columns']) == 0: assert len(choice_hyperparameter.value_range) == 1 - assert 'NoScaler' in choice_hyperparameter.value_range, \ - "Provided {} in choices, however, the dataset " \ - "is incompatible with it".format(choice_hyperparameter.value_range) + if 'NoScaler' not in choice_hyperparameter.value_range: + raise ValueError("Provided {} in choices, however, the dataset " \ + "is incompatible with it".format(choice_hyperparameter.value_range)) + preprocessor = CSH.CategoricalHyperparameter('__choice__', choice_hyperparameter.value_range, default_value=choice_hyperparameter.default_value) diff --git a/autoPyTorch/pipeline/components/setup/network_embedding/base_network_embedding_choice.py b/autoPyTorch/pipeline/components/setup/network_embedding/base_network_embedding_choice.py index 45d15f06a..14a5c93d9 100644 --- a/autoPyTorch/pipeline/components/setup/network_embedding/base_network_embedding_choice.py +++ b/autoPyTorch/pipeline/components/setup/network_embedding/base_network_embedding_choice.py @@ -163,9 +163,9 @@ def get_hyperparameter_search_space( choice_hyperparameter.value_range)) if len(dataset_properties['categorical_columns']) == 0: assert len(choice_hyperparameter.value_range) == 1 - assert 'NoEmbedding' in choice_hyperparameter.value_range, \ - "Provided {} in choices, however, the dataset " \ - "is incompatible with it".format(choice_hyperparameter.value_range) + if 'NoEmbedding' not in choice_hyperparameter.value_range: + raise ValueError("Provided {} in choices, however, the dataset " + "is incompatible with it".format(choice_hyperparameter.value_range)) embedding = CSH.CategoricalHyperparameter('__choice__', choice_hyperparameter.value_range, default_value=choice_hyperparameter.default_value) diff --git a/autoPyTorch/pipeline/tabular_classification.py b/autoPyTorch/pipeline/tabular_classification.py index 752d7c03a..1ca0635b6 100644 --- a/autoPyTorch/pipeline/tabular_classification.py +++ b/autoPyTorch/pipeline/tabular_classification.py @@ -189,8 +189,7 @@ def _get_hyperparameter_search_space(self, cs=cs, dataset_properties=dataset_properties, exclude=exclude, include=include, pipeline=self.steps) - # Here we add custom code, like this with this - # is not a valid configuration + # Here we add custom code, that is used to ensure valid configurations, For example # Learned Entity Embedding is only valid when encoder is one hot encoder if 'network_embedding' in self.named_steps.keys() and 'encoder' in self.named_steps.keys(): embeddings = cs.get_hyperparameter('network_embedding:__choice__').choices diff --git a/autoPyTorch/utils/common.py b/autoPyTorch/utils/common.py index a1227d560..98bd20a68 100644 --- a/autoPyTorch/utils/common.py +++ b/autoPyTorch/utils/common.py @@ -28,9 +28,11 @@ class FitRequirement(NamedTuple): requirements have to be user specified or are generated by the pipeline itself. Attributes: - name: The name of the variable expected in the input dictionary - supported_types: An iterable of all types that are supported - user_defined: If false, this requirement does not have to be given to the pipeline + name (str): The name of the variable expected in the input dictionary + supported_types (Iterable[Type]): An iterable of all types that are supported + user_defined (bool): If false, this requirement does not have to be given to the pipeline + dataset_property (bool): If True, this requirement is automatically inferred + by the Dataset class """ name: str @@ -53,7 +55,9 @@ class HyperparameterSearchSpace(NamedTuple): hyperparameter (str): name of the hyperparameter value_range (Sequence[HyperparameterValueType]): - range of the hyperparameter + range of the hyperparameter, can be defined as min and + max values for Numerical hyperparameter or a list of + choices for a Categorical hyperparameter default_value (HyperparameterValueType): default value of the hyperparameter log (bool): @@ -64,6 +68,13 @@ class HyperparameterSearchSpace(NamedTuple): default_value: HyperparameterValueType log: bool = False + def __str__(self) -> str: + """ + String representation for the Search Space + """ + return "Hyperparameter: %s | Range: %s | Default: %s | log: %s" % ( + self.hyperparameter, self.value_range, self.default_value, self.log) + def replace_prefix_in_config_dict(config: Dict[str, Any], prefix: str, replace: str = "") -> Dict[str, Any]: """ diff --git a/test/test_pipeline/test_tabular_classification.py b/test/test_pipeline/test_tabular_classification.py index b752e52da..fb821d9ec 100644 --- a/test/test_pipeline/test_tabular_classification.py +++ b/test/test_pipeline/test_tabular_classification.py @@ -343,6 +343,8 @@ def test_set_choices_updates(self, fit_dictionary_tabular): continue name = name.split(':') hyperparameter_name = ':'.join(name[1:]) + # Using NoEmbedding is safer for this test + # to avoid forbidden configuration errors if name[0] == 'network_embedding' and hyperparameter_name == '__choice__': value_range = ('NoEmbedding',) default_value = 'NoEmbedding' @@ -359,9 +361,17 @@ def test_set_choices_updates(self, fit_dictionary_tabular): @pytest.mark.parametrize("fit_dictionary_tabular", ['iris'], indirect=True) def test_constant_pipeline_iris(fit_dictionary_tabular): search_space_updates = HyperparameterSearchSpaceUpdates() + search_space_updates.append(node_name='feature_preprocessor', + hyperparameter='__choice__', + value_range=['PolynomialFeatures'], + default_value='PolynomialFeatures') + search_space_updates.append(node_name='scaler', + hyperparameter='__choice__', + value_range=['StandardScaler'], + default_value='StandardScaler') search_space_updates.append(node_name='network_backbone', hyperparameter='__choice__', - value_range=['MLPBackbone'], + value_range=['MLPBackbone', 'ShapedMLPBackbone'], default_value='MLPBackbone') search_space_updates.append(node_name='network_backbone', hyperparameter='MLPBackbone:num_groups', @@ -390,11 +400,25 @@ def test_constant_pipeline_iris(fit_dictionary_tabular): pipeline = TabularClassificationPipeline(dataset_properties=fit_dictionary_tabular['dataset_properties'], search_space_updates=search_space_updates) + fit_dictionary_tabular['additional_metrics'] = ['balanced_accuracy'] + try: pipeline.fit(fit_dictionary_tabular) except Exception as e: pytest.fail(f"Failed due to {e}") + configuration = pipeline.configuration + + assert 'PolynomialFeatures' == configuration.get('feature_preprocessor:__choice__') + assert 'StandardScaler' == configuration.get('scaler:__choice__') + assert 'MLPBackbone' == configuration.get('network_backbone:__choice__') + assert 'StandardTrainer' == configuration.get('trainer:__choice__') + assert 'NoScheduler' == configuration.get('lr_scheduler:__choice__') + assert 'AdamOptimizer' == configuration.get('optimizer:__choice__') + assert 1 == configuration.get('network_backbone:MLPBackbone:num_groups') + assert 100 == configuration.get('network_backbone:MLPBackbone:num_units_1') + assert 1e-2 == configuration.get('optimizer:AdamOptimizer:lr') + # To make sure we fitted the model, there should be a # run summary object with accuracy run_summary = pipeline.named_steps['trainer'].run_summary @@ -403,12 +427,14 @@ def test_constant_pipeline_iris(fit_dictionary_tabular): # Make sure that performance was properly captured assert run_summary.performance_tracker['train_loss'][1] > 0 assert run_summary.total_parameter_count > 0 - assert 'accuracy' in run_summary.performance_tracker['train_metrics'][1] + assert 'balanced_accuracy' in run_summary.performance_tracker['train_metrics'][1] # Make sure default pipeline achieves a good score for dummy datasets epoch2loss = run_summary.performance_tracker['val_loss'] best_loss = min(list(epoch2loss.values())) epoch_where_best = list(epoch2loss.keys())[list(epoch2loss.values()).index(best_loss)] - score = run_summary.performance_tracker['val_metrics'][epoch_where_best]['accuracy'] + val_score = run_summary.performance_tracker['val_metrics'][epoch_where_best]['balanced_accuracy'] + train_score = run_summary.performance_tracker['train_metrics'][epoch_where_best]['balanced_accuracy'] - assert score >= 0.9, run_summary.performance_tracker['val_metrics'] + assert val_score >= 0.9, run_summary.performance_tracker['val_metrics'] + assert train_score >= 0.9, run_summary.performance_tracker['train_metrics'] \ No newline at end of file diff --git a/test/test_utils/test_hyperparameter_search_space_update.py b/test/test_utils/test_hyperparameter_search_space_update.py new file mode 100644 index 000000000..a86e50827 --- /dev/null +++ b/test/test_utils/test_hyperparameter_search_space_update.py @@ -0,0 +1,83 @@ +from ConfigSpace.configuration_space import ConfigurationSpace +from ConfigSpace.hyperparameters import ( + CategoricalHyperparameter, + UniformFloatHyperparameter, + UniformIntegerHyperparameter +) +from autoPyTorch.pipeline.components.base_component import autoPyTorchComponent +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter +from autoPyTorch.utils.hyperparameter_search_space_update import HyperparameterSearchSpaceUpdates + + +class DummyComponent(autoPyTorchComponent): + def __init__(self): + self._cs_updates = {} + + @staticmethod + def get_hyperparameter_search_space( + dataset_properties=None, + X=HyperparameterSearchSpace("X", + value_range=[-5, 5], + default_value=0), + Y=HyperparameterSearchSpace("Y", + value_range=[0, 1], + default_value=0), + Z=HyperparameterSearchSpace("Z", + value_range=['a', 'b', 1], + default_value='a'), + ): + cs = ConfigurationSpace() + add_hyperparameter(cs, X, UniformIntegerHyperparameter) + add_hyperparameter(cs, Y, UniformFloatHyperparameter) + add_hyperparameter(cs, Z, CategoricalHyperparameter) + return cs + + +def test_hyperparameter_search_space_update(): + updates = HyperparameterSearchSpaceUpdates() + updates.append(node_name="dummy_node", + hyperparameter="X", + value_range=[1, 3], + default_value=2) + updates.append(node_name="dummy_node", + hyperparameter="Y", + value_range=[0.1, 0.5], + default_value=0.1, + log=True) + updates.append(node_name="dummy_node", + hyperparameter="Z", + value_range=['a', 3], + default_value=3) + dummy_component = DummyComponent() + updates.apply([("dummy_node", dummy_component)]) + new_updates = dummy_component._get_search_space_updates() + config_space = dummy_component.get_hyperparameter_search_space(**new_updates) + + for i, (update, hp_type) in enumerate(zip(['X', 'Y', 'Z'], + [UniformIntegerHyperparameter, + UniformFloatHyperparameter, + CategoricalHyperparameter])): + + search_space_update = updates.updates[i] + search_space = search_space_update.get_search_space() + + assert search_space.hyperparameter == search_space_update.hyperparameter + assert search_space.value_range == search_space_update.value_range + assert search_space.default_value == search_space_update.default_value + assert search_space.log == search_space_update.log + + assert update in dummy_component._cs_updates + assert update in new_updates + assert update in config_space + + hp = config_space.get_hyperparameter(update) + assert isinstance(hp, hp_type) + + if update == 'Z': + assert all(a == b for a, b in zip(hp.choices, search_space.value_range)) + else: + assert hp.lower == search_space.value_range[0] + assert hp.upper == search_space.value_range[1] + assert hp.log == search_space.log + + assert hp.default_value == search_space.default_value From 5fcc105c8a967fc2d23d7571fba7d8edb14baf47 Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Wed, 24 Mar 2021 16:53:26 +0100 Subject: [PATCH 10/10] Fix flake --- .../tabular_preprocessing/scaling/base_scaler_choice.py | 2 +- test/test_pipeline/test_tabular_classification.py | 2 +- test/test_utils/test_hyperparameter_search_space_update.py | 7 ++++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/base_scaler_choice.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/base_scaler_choice.py index 8e32c0cd5..7c5f22fd5 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/base_scaler_choice.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/base_scaler_choice.py @@ -81,7 +81,7 @@ def get_hyperparameter_search_space(self, if len(dataset_properties['numerical_columns']) == 0: assert len(choice_hyperparameter.value_range) == 1 if 'NoScaler' not in choice_hyperparameter.value_range: - raise ValueError("Provided {} in choices, however, the dataset " \ + raise ValueError("Provided {} in choices, however, the dataset " "is incompatible with it".format(choice_hyperparameter.value_range)) preprocessor = CSH.CategoricalHyperparameter('__choice__', diff --git a/test/test_pipeline/test_tabular_classification.py b/test/test_pipeline/test_tabular_classification.py index fb821d9ec..108a217e7 100644 --- a/test/test_pipeline/test_tabular_classification.py +++ b/test/test_pipeline/test_tabular_classification.py @@ -437,4 +437,4 @@ def test_constant_pipeline_iris(fit_dictionary_tabular): train_score = run_summary.performance_tracker['train_metrics'][epoch_where_best]['balanced_accuracy'] assert val_score >= 0.9, run_summary.performance_tracker['val_metrics'] - assert train_score >= 0.9, run_summary.performance_tracker['train_metrics'] \ No newline at end of file + assert train_score >= 0.9, run_summary.performance_tracker['train_metrics'] diff --git a/test/test_utils/test_hyperparameter_search_space_update.py b/test/test_utils/test_hyperparameter_search_space_update.py index a86e50827..ee4af4814 100644 --- a/test/test_utils/test_hyperparameter_search_space_update.py +++ b/test/test_utils/test_hyperparameter_search_space_update.py @@ -4,6 +4,7 @@ UniformFloatHyperparameter, UniformIntegerHyperparameter ) + from autoPyTorch.pipeline.components.base_component import autoPyTorchComponent from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter from autoPyTorch.utils.hyperparameter_search_space_update import HyperparameterSearchSpaceUpdates @@ -54,9 +55,9 @@ def test_hyperparameter_search_space_update(): config_space = dummy_component.get_hyperparameter_search_space(**new_updates) for i, (update, hp_type) in enumerate(zip(['X', 'Y', 'Z'], - [UniformIntegerHyperparameter, - UniformFloatHyperparameter, - CategoricalHyperparameter])): + [UniformIntegerHyperparameter, + UniformFloatHyperparameter, + CategoricalHyperparameter])): search_space_update = updates.updates[i] search_space = search_space_update.get_search_space()