diff --git a/autoPyTorch/pipeline/base_pipeline.py b/autoPyTorch/pipeline/base_pipeline.py index 2e70478a3..792268487 100644 --- a/autoPyTorch/pipeline/base_pipeline.py +++ b/autoPyTorch/pipeline/base_pipeline.py @@ -398,24 +398,47 @@ def _check_search_space_updates(self, include: Optional[Dict[str, Any]], raise ValueError("Unknown node name. Expected update node name to be in {} " "got {}".format(self.named_steps.keys(), update.node_name)) node = self.named_steps[update.node_name] + # if node is a choice module if hasattr(node, 'get_components'): split_hyperparameter = update.hyperparameter.split(':') + # check if component is not present in include if include is not None and update.node_name in include.keys(): if split_hyperparameter[0] not in include[update.node_name]: raise ValueError("Not found {} in include".format(split_hyperparameter[0])) + # check if component is present in exclude if exclude is not None and update.node_name in exclude.keys(): if split_hyperparameter[0] in exclude[update.node_name]: raise ValueError("Found {} in exclude".format(split_hyperparameter[0])) components = node.get_components() - if split_hyperparameter[0] not in components.keys(): + # if hyperparameter is __choice__, check if + # the components in the value range of search space update + # are in components of the choice module + if split_hyperparameter[0] == '__choice__': + for choice in update.value_range: + if include is not None and update.node_name in include.keys(): + if choice not in include[update.node_name]: + raise ValueError("Not found {} in include".format(choice)) + if exclude is not None and update.node_name in exclude.keys(): + if choice in exclude[update.node_name]: + raise ValueError("Found {} in exclude".format(choice)) + if choice not in components.keys(): + raise ValueError("Unknown hyperparameter for choice {}. " + "Expected update hyperparameter " + "to be in {} got {}".format(node.__class__.__name__, + components.keys(), choice)) + # check if the component whose hyperparameter + # needs to be updated is in components of the + # choice module + elif split_hyperparameter[0] not in components.keys(): raise ValueError("Unknown hyperparameter for choice {}. " "Expected update hyperparameter " "to be in {} got {}".format(node.__class__.__name__, components.keys(), split_hyperparameter[0])) else: + # check if hyperparameter is in the search space of the component component = components[split_hyperparameter[0]] if split_hyperparameter[1] not in component. \ get_hyperparameter_search_space(dataset_properties=self.dataset_properties): diff --git a/autoPyTorch/pipeline/components/base_choice.py b/autoPyTorch/pipeline/components/base_choice.py index b6fffbaf2..53e94e9f3 100644 --- a/autoPyTorch/pipeline/components/base_choice.py +++ b/autoPyTorch/pipeline/components/base_choice.py @@ -1,6 +1,7 @@ +import re import warnings from collections import OrderedDict -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional from ConfigSpace.configuration_space import Configuration, ConfigurationSpace @@ -9,7 +10,8 @@ from sklearn.utils import check_random_state from autoPyTorch.pipeline.components.base_component import autoPyTorchComponent -from autoPyTorch.utils.common import FitRequirement +from autoPyTorch.utils.common import FitRequirement, HyperparameterSearchSpace +from autoPyTorch.utils.hyperparameter_search_space_update import HyperparameterSearchSpaceUpdate class autoPyTorchChoice(object): @@ -49,7 +51,7 @@ def __init__(self, # self.set_hyperparameters(self.configuration) self.choice: Optional[autoPyTorchComponent] = None - self._cs_updates: Dict[str, Tuple] = dict() + self._cs_updates: Dict[str, HyperparameterSearchSpaceUpdate] = dict() def get_fit_requirements(self) -> Optional[List[FitRequirement]]: if self.choice is not None: @@ -247,35 +249,35 @@ def _check_dataset_properties(self, dataset_properties: Dict[str, Any]) -> None: """ assert isinstance(dataset_properties, dict), "dataset_properties must be a dictionary" - def _apply_search_space_update(self, name: str, new_value_range: Union[List, Tuple], - default_value: Union[int, float, str], log: bool = False) -> None: - """Allows the user to update a hyperparameter + def _apply_search_space_update(self, hyperparameter_search_space_update: HyperparameterSearchSpaceUpdate) -> None: + """ + Applies search space update to the class - Arguments: - name {string} -- name of hyperparameter - new_value_range {List[?] -- value range can be either lower, upper or a list of possible conditionals - log {bool} -- is hyperparameter logscale + Args: + hyperparameter_search_space_update (HyperparameterSearchSpaceUpdate): + Search Space update for the current autoPyTorchChoice module """ - if len(new_value_range) == 0: - raise ValueError("The new value range needs at least one value") - self._cs_updates[name] = tuple([new_value_range, default_value, log]) + self._cs_updates[hyperparameter_search_space_update.hyperparameter] = hyperparameter_search_space_update - def _get_search_space_updates(self, prefix: Optional[str] = None) -> Dict[str, Tuple]: + def _get_search_space_updates(self, prefix: Optional[str] = None) -> Dict[str, HyperparameterSearchSpace]: """Get the search space updates with the given prefix - Keyword Arguments: - prefix {str} -- Only return search space updates with given prefix (default: {None}) + Args: + prefix (str): + Only return search space updates with given prefix (default: {None}) Returns: - dict -- Mapping of search space updates. Keys don't contain the prefix. + Dict[str, HyperparameterSearchSpace]: + Mapping of search space updates. Keys don't contain the prefix. """ - if prefix is None: - return self._cs_updates - result: Dict[str, Tuple] = dict() - # iterate over all search space updates of this node and filter the ones out, that have the given prefix + result: Dict[str, HyperparameterSearchSpace] = dict() + + # iterate over all search space updates of this node and keep the ones that have the given prefix for key in self._cs_updates.keys(): - if key.startswith(prefix): - result[key[len(prefix) + 1:]] = self._cs_updates[key] + if prefix is None: + result[key] = self._cs_updates[key].get_search_space() + elif re.search(f'^{prefix}', key) is not None: + result[key[len(prefix) + 1:]] = self._cs_updates[key].get_search_space(remove_prefix=prefix) return result diff --git a/autoPyTorch/pipeline/components/base_component.py b/autoPyTorch/pipeline/components/base_component.py index 80fb1ddd0..09d981342 100644 --- a/autoPyTorch/pipeline/components/base_component.py +++ b/autoPyTorch/pipeline/components/base_component.py @@ -4,19 +4,20 @@ import sys import warnings from collections import OrderedDict -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional from ConfigSpace.configuration_space import Configuration, ConfigurationSpace from sklearn.base import BaseEstimator -from autoPyTorch.utils.common import FitRequirement +from autoPyTorch.utils.common import FitRequirement, HyperparameterSearchSpace +from autoPyTorch.utils.hyperparameter_search_space_update import HyperparameterSearchSpaceUpdate def find_components( - package: str, - directory: str, - base_class: BaseEstimator + package: str, + directory: str, + base_class: BaseEstimator ) -> Dict[str, BaseEstimator]: """Utility to find component on a given directory, that inherit from base_class @@ -34,8 +35,7 @@ def find_components( module = importlib.import_module(full_module_name) for member_name, obj in inspect.getmembers(module): - if inspect.isclass(obj) and issubclass(obj, base_class) and \ - obj != base_class: + if inspect.isclass(obj) and issubclass(obj, base_class) and obj != base_class: # TODO test if the obj implements the interface # Keep in mind that this only instantiates the ensemble_wrapper, # but not the real target classifier @@ -96,7 +96,7 @@ class autoPyTorchComponent(BaseEstimator): def __init__(self) -> None: super().__init__() self._fit_requirements: List[FitRequirement] = list() - self._cs_updates: Dict[str, Tuple] = dict() + self._cs_updates: Dict[str, HyperparameterSearchSpaceUpdate] = dict() @classmethod def get_required_properties(cls) -> Optional[List[str]]: @@ -140,7 +140,7 @@ def get_properties(dataset_properties: Optional[Dict[str, str]] = None @staticmethod def get_hyperparameter_search_space( - dataset_properties: Optional[Dict[str, str]] = None + dataset_properties: Optional[Dict[str, str]] = None ) -> ConfigurationSpace: """Return the configuration space of this classification algorithm. @@ -253,8 +253,7 @@ def __str__(self) -> str: name = self.get_properties()['name'] return "autoPyTorch.pipeline %s" % name - def _apply_search_space_update(self, name: str, new_value_range: Union[List, Tuple], - default_value: Union[int, float, str], log: bool = False) -> None: + def _apply_search_space_update(self, hyperparameter_search_space_update: HyperparameterSearchSpaceUpdate) -> None: """Allows the user to update a hyperparameter Arguments: @@ -263,26 +262,18 @@ def _apply_search_space_update(self, name: str, new_value_range: Union[List, Tup log {bool} -- is hyperparameter logscale """ - if len(new_value_range) == 0: - raise ValueError("The new value range needs at least one value") - self._cs_updates[name] = tuple([new_value_range, default_value, log]) + self._cs_updates[hyperparameter_search_space_update.hyperparameter] = hyperparameter_search_space_update - def _get_search_space_updates(self, prefix: Optional[str] = None) -> Dict[str, Tuple]: - """Get the search space updates with the given prefix - - Keyword Arguments: - prefix {str} -- Only return search space updates with given prefix (default: {None}) + def _get_search_space_updates(self) -> Dict[str, HyperparameterSearchSpace]: + """Get the search space updates Returns: dict -- Mapping of search space updates. Keys don't contain the prefix. """ - if prefix is None: - return self._cs_updates - result: Dict[str, Tuple] = dict() + + result: Dict[str, HyperparameterSearchSpace] = dict() # iterate over all search space updates of this node and keep the ones that have the given prefix for key in self._cs_updates.keys(): - if key.startswith(prefix): - # different for autopytorch component as the hyperparameter - result[key[len(prefix):]] = self._cs_updates[key] + result[key] = self._cs_updates[key].get_search_space() return result diff --git a/autoPyTorch/pipeline/components/preprocessing/image_preprocessing/normalise/base_normalizer_choice.py b/autoPyTorch/pipeline/components/preprocessing/image_preprocessing/normalise/base_normalizer_choice.py index 696601b4f..acc9f5b64 100644 --- a/autoPyTorch/pipeline/components/preprocessing/image_preprocessing/normalise/base_normalizer_choice.py +++ b/autoPyTorch/pipeline/components/preprocessing/image_preprocessing/normalise/base_normalizer_choice.py @@ -76,10 +76,21 @@ def get_hyperparameter_search_space(self, default = default_ break - preprocessor = CSH.CategoricalHyperparameter('__choice__', - list(available_preprocessors.keys()), - default_value=default) - + updates = self._get_search_space_updates() + if '__choice__' in updates.keys(): + choice_hyperparameter = updates['__choice__'] + if not set(choice_hyperparameter.value_range).issubset(available_preprocessors): + raise ValueError("Expected given update for {} to have " + "choices in {} got {}".format(self.__class__.__name__, + available_preprocessors, + choice_hyperparameter.value_range)) + preprocessor = CSH.CategoricalHyperparameter('__choice__', + choice_hyperparameter.value_range, + default_value=choice_hyperparameter.default_value) + else: + preprocessor = CSH.CategoricalHyperparameter('__choice__', + list(available_preprocessors.keys()), + default_value=default) cs.add_hyperparameter(preprocessor) # add only child hyperparameters of early_preprocessor choices diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/encoding/base_encoder_choice.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/encoding/base_encoder_choice.py index df71ff209..7ddbf8eaf 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/encoding/base_encoder_choice.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/encoding/base_encoder_choice.py @@ -75,19 +75,37 @@ def get_hyperparameter_search_space(self, default = default_ break - # add only no encoder to choice hyperparameters in case the dataset is only numerical - if len(dataset_properties['categorical_columns']) == 0: - default = 'NoEncoder' - if include is not None and default not in include: - raise ValueError("Provided {} in include, however, the dataset " - "is incompatible with it".format(include)) + updates = self._get_search_space_updates() + if '__choice__' in updates.keys(): + choice_hyperparameter = updates['__choice__'] + if not set(choice_hyperparameter.value_range).issubset(available_preprocessors): + raise ValueError("Expected given update for {} to have " + "choices in {} got {}".format(self.__class__.__name__, + available_preprocessors, + choice_hyperparameter.value_range)) + if len(dataset_properties['categorical_columns']) == 0: + assert len(choice_hyperparameter.value_range) == 1 + assert 'NoEncoder' in choice_hyperparameter.value_range, \ + "Provided {} in choices, however, the dataset " \ + "is incompatible with it".format(choice_hyperparameter.value_range) + preprocessor = CSH.CategoricalHyperparameter('__choice__', - ['NoEncoder'], - default_value=default) + choice_hyperparameter.value_range, + default_value=choice_hyperparameter.default_value) else: - preprocessor = CSH.CategoricalHyperparameter('__choice__', - list(available_preprocessors.keys()), - default_value=default) + # add only no encoder to choice hyperparameters in case the dataset is only numerical + if len(dataset_properties['categorical_columns']) == 0: + default = 'NoEncoder' + if include is not None and default not in include: + raise ValueError("Provided {} in include, however, the dataset " + "is incompatible with it".format(include)) + preprocessor = CSH.CategoricalHyperparameter('__choice__', + ['NoEncoder'], + default_value=default) + else: + preprocessor = CSH.CategoricalHyperparameter('__choice__', + list(available_preprocessors.keys()), + default_value=default) cs.add_hyperparameter(preprocessor) diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/KernelPCA.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/KernelPCA.py index 4fb77b90f..2a4737c4d 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/KernelPCA.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/KernelPCA.py @@ -1,5 +1,5 @@ from math import ceil, floor -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, Optional, Union from ConfigSpace.conditions import EqualsCondition, InCondition from ConfigSpace.configuration_space import ConfigurationSpace @@ -14,9 +14,9 @@ import sklearn.decomposition from sklearn.base import BaseEstimator -from autoPyTorch.pipeline.components.preprocessing.tabular_preprocessing.feature_preprocessing.\ +from autoPyTorch.pipeline.components.preprocessing.tabular_preprocessing.feature_preprocessing. \ base_feature_preprocessor import autoPyTorchFeaturePreprocessingComponent -from autoPyTorch.utils.common import FitRequirement +from autoPyTorch.utils.common import FitRequirement, HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter class KernelPCA(autoPyTorchFeaturePreprocessingComponent): @@ -48,36 +48,56 @@ def fit(self, X: Dict[str, Any], y: Any = None) -> BaseEstimator: @staticmethod def get_hyperparameter_search_space( dataset_properties: Optional[Dict[str, str]] = None, - n_components: Tuple[Tuple, float] = ((0.5, 0.9), 0.5), - kernel: Tuple[Tuple, str] = (('poly', 'rbf', 'sigmoid', 'cosine'), 'rbf'), - gamma: Tuple[Tuple, float, bool] = ((3.0517578125e-05, 8), 0.01, True), - degree: Tuple[Tuple, int] = ((2, 5), 3), - coef0: Tuple[Tuple, float] = ((-1, 1), 0) + n_components: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='n_components', + value_range=(0.5, 0.9), + default_value=0.5, + ), + kernel: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='kernel', + value_range=('poly', 'rbf', 'sigmoid', 'cosine'), + default_value='rbf', + ), + gamma: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='gamma', + value_range=(3.0517578125e-05, 8), + default_value=0.01, + log=True), + degree: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='degree', + value_range=(2, 5), + default_value=3, + log=True), + coef0: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='coef0', + value_range=(-1, 1), + default_value=0, + ) ) -> ConfigurationSpace: + cs = ConfigurationSpace() + if dataset_properties is not None: n_features = len(dataset_properties['numerical_columns']) - n_components = ((floor(n_components[0][0] * n_features), ceil(n_components[0][1] * n_features)), - ceil(n_components[1] * n_features)) + if n_features == 1: + log = False + else: + log = n_components.log + n_components = HyperparameterSearchSpace(hyperparameter='n_components', + value_range=( + floor(float(n_components.value_range[0]) * n_features), + ceil(float(n_components.value_range[1]) * n_features)), + default_value=ceil(float(n_components.default_value) * n_features), + log=log) else: - n_components = ((10, 2000), 100) - - n_components = UniformIntegerHyperparameter( - "n_components", lower=n_components[0][0], upper=n_components[0][1], default_value=n_components[1]) - kernel_hp = CategoricalHyperparameter('kernel', choices=kernel[0], default_value=kernel[1]) - gamma = UniformFloatHyperparameter( - "gamma", - lower=gamma[0][0], upper=gamma[0][1], - log=gamma[2], - default_value=gamma[1], - ) - coef0 = UniformFloatHyperparameter("coef0", lower=coef0[0][0], upper=coef0[0][1], default_value=coef0[1]) - cs = ConfigurationSpace() - cs.add_hyperparameters([n_components, kernel_hp, gamma, coef0]) + n_components = HyperparameterSearchSpace(hyperparameter='n_components', + value_range=(10, 2000), + default_value=100, + log=n_components.log) + + add_hyperparameter(cs, n_components, UniformIntegerHyperparameter) + kernel_hp = get_hyperparameter(kernel, CategoricalHyperparameter) + gamma = get_hyperparameter(gamma, UniformFloatHyperparameter) + coef0 = get_hyperparameter(coef0, UniformFloatHyperparameter) + cs.add_hyperparameters([kernel_hp, gamma, coef0]) if "poly" in kernel_hp.choices: - degree = UniformIntegerHyperparameter('degree', lower=degree[0][0], upper=degree[0][1], - default_value=degree[1]) + degree = get_hyperparameter(degree, UniformIntegerHyperparameter) cs.add_hyperparameters([degree]) degree_depends_on_poly = EqualsCondition(degree, kernel_hp, "poly") cs.add_conditions([degree_depends_on_poly]) diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/Nystroem.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/Nystroem.py index 93be983e9..0a8f6c63d 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/Nystroem.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/Nystroem.py @@ -1,5 +1,5 @@ from math import ceil, floor -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, Optional, Union from ConfigSpace.conditions import EqualsCondition, InCondition from ConfigSpace.configuration_space import ConfigurationSpace @@ -14,8 +14,9 @@ import sklearn.kernel_approximation from sklearn.base import BaseEstimator -from autoPyTorch.pipeline.components.preprocessing.tabular_preprocessing.feature_preprocessing.\ +from autoPyTorch.pipeline.components.preprocessing.tabular_preprocessing.feature_preprocessing. \ base_feature_preprocessor import autoPyTorchFeaturePreprocessingComponent +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter class Nystroem(autoPyTorchFeaturePreprocessingComponent): @@ -44,41 +45,60 @@ def fit(self, X: Dict[str, Any], y: Any = None) -> BaseEstimator: @staticmethod def get_hyperparameter_search_space( dataset_properties: Optional[Dict[str, str]] = None, - n_components: Tuple[Tuple, float, bool] = ((0.5, 0.9), 0.5, True), - kernel: Tuple[Tuple, str] = (('poly', 'rbf', 'sigmoid', 'cosine'), 'rbf'), - gamma: Tuple[Tuple, float, bool] = ((3.0517578125e-05, 8), 0.01, True), - degree: Tuple[Tuple, int] = ((2, 5), 3), - coef0: Tuple[Tuple, float] = ((-1, 1), 0) + n_components: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='n_components', + value_range=(0.5, 0.9), + default_value=0.5, + ), + kernel: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='kernel', + value_range=('poly', 'rbf', 'sigmoid', 'cosine'), + default_value='rbf', + ), + gamma: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='gamma', + value_range=(3.0517578125e-05, 8), + default_value=0.01, + log=True), + degree: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='degree', + value_range=(2, 5), + default_value=3, + log=True), + coef0: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='coef0', + value_range=(-1, 1), + default_value=0, + ) ) -> ConfigurationSpace: + cs = ConfigurationSpace() + if dataset_properties is not None: n_features = len(dataset_properties['numerical_columns']) # if numerical features are 1, set log to False if n_features == 1: log = False else: - log = n_components[2] - n_components = ((floor(n_components[0][0] * n_features), ceil(n_components[0][1] * n_features)), - ceil(n_components[1] * n_features), log) + log = n_components.log + n_components = HyperparameterSearchSpace(hyperparameter='n_components', + value_range=( + floor(float(n_components.value_range[0]) * n_features), + ceil(float(n_components.value_range[1]) * n_features)), + default_value=ceil(float(n_components.default_value) * n_features), + log=log) else: - n_components = ((10, 2000), 100, True) + n_components = HyperparameterSearchSpace(hyperparameter='n_components', + value_range=(10, 2000), + default_value=100, + log=n_components.log) - n_components = UniformIntegerHyperparameter( - "n_components", lower=n_components[0][0], upper=n_components[0][1], - default_value=n_components[1], log=n_components[2]) - kernel_hp = CategoricalHyperparameter('kernel', choices=kernel[0], default_value=kernel[1]) - gamma = UniformFloatHyperparameter( - "gamma", - lower=gamma[0][0], upper=gamma[0][1], - log=gamma[2], - default_value=gamma[1], - ) - degree = UniformIntegerHyperparameter('degree', lower=degree[0][0], upper=degree[0][1], default_value=degree[1]) - coef0 = UniformFloatHyperparameter("coef0", lower=coef0[0][0], upper=coef0[0][1], default_value=coef0[1]) - cs = ConfigurationSpace() - cs.add_hyperparameters([n_components, kernel_hp, degree, gamma, coef0]) + add_hyperparameter(cs, n_components, UniformIntegerHyperparameter) + kernel_hp = get_hyperparameter(kernel, CategoricalHyperparameter) + gamma = get_hyperparameter(gamma, UniformFloatHyperparameter) + coef0 = get_hyperparameter(coef0, UniformFloatHyperparameter) + cs.add_hyperparameters([kernel_hp, gamma, coef0]) - degree_depends_on_poly = EqualsCondition(degree, kernel_hp, "poly") + if "poly" in kernel_hp.choices: + degree = get_hyperparameter(degree, UniformIntegerHyperparameter) + cs.add_hyperparameters([degree]) + degree_depends_on_poly = EqualsCondition(degree, kernel_hp, "poly") + cs.add_conditions([degree_depends_on_poly]) kernels = [] if "sigmoid" in kernel_hp.choices: kernels.append("sigmoid") @@ -91,7 +111,7 @@ def get_hyperparameter_search_space( if "poly" in kernel_hp.choices: kernels.append("poly") gamma_condition = InCondition(gamma, kernel_hp, kernels) - cs.add_conditions([degree_depends_on_poly, coef0_condition, gamma_condition]) + cs.add_conditions([coef0_condition, gamma_condition]) return cs @staticmethod diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/PolynomialFeatures.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/PolynomialFeatures.py index 9f542acd0..a41c0a26d 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/PolynomialFeatures.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/PolynomialFeatures.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, Optional, Union from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( @@ -11,8 +11,9 @@ import sklearn.decomposition from sklearn.base import BaseEstimator -from autoPyTorch.pipeline.components.preprocessing.tabular_preprocessing.feature_preprocessing.\ +from autoPyTorch.pipeline.components.preprocessing.tabular_preprocessing.feature_preprocessing. \ base_feature_preprocessor import autoPyTorchFeaturePreprocessingComponent +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter class PolynomialFeatures(autoPyTorchFeaturePreprocessingComponent): @@ -27,7 +28,6 @@ def __init__(self, degree: int = 2, interaction_only: bool = False, super().__init__() def fit(self, X: Dict[str, Any], y: Any = None) -> BaseEstimator: - self.preprocessor['numerical'] = sklearn.preprocessing.PolynomialFeatures( degree=self.degree, interaction_only=self.interaction_only, include_bias=self.include_bias) @@ -42,20 +42,22 @@ def get_properties(dataset_properties: Optional[Dict[str, str]] = None) -> Dict[ @staticmethod def get_hyperparameter_search_space( dataset_properties: Optional[Dict[str, str]] = None, - degree: Tuple[Tuple, int] = ((2, 3), 2), - interaction_only: Tuple[Tuple, bool] = ((True, False), False), - include_bias: Tuple[Tuple, bool] = ((True, False), False) + degree: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='degree', + value_range=(2, 3), + default_value=2, + log=True), + interaction_only: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='interaction_only', + value_range=(True, False), + default_value=False, + ), + include_bias: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='include_bias', + value_range=(True, False), + default_value=False, + ), ) -> ConfigurationSpace: - - degree = UniformIntegerHyperparameter("degree", lower=degree[0][0], upper=degree[0][1], default_value=degree[1]) - interaction_only = CategoricalHyperparameter("interaction_only", - choices=interaction_only[0], - default_value=interaction_only[1]) - include_bias = CategoricalHyperparameter("include_bias", - choices=include_bias[0], - default_value=include_bias[1]) - cs = ConfigurationSpace() - cs.add_hyperparameters([degree, interaction_only, include_bias]) + add_hyperparameter(cs, degree, UniformIntegerHyperparameter) + add_hyperparameter(cs, interaction_only, CategoricalHyperparameter) + add_hyperparameter(cs, include_bias, CategoricalHyperparameter) return cs diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/PowerTransformer.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/PowerTransformer.py index c02606c3d..767a0f6c1 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/PowerTransformer.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/PowerTransformer.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, Optional, Union from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( @@ -12,6 +12,7 @@ from autoPyTorch.pipeline.components.preprocessing.tabular_preprocessing.feature_preprocessing. \ base_feature_preprocessor import autoPyTorchFeaturePreprocessingComponent +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter class PowerTransformer(autoPyTorchFeaturePreprocessingComponent): @@ -36,14 +37,13 @@ def get_properties(dataset_properties: Optional[Dict[str, str]] = None) -> Dict[ @staticmethod def get_hyperparameter_search_space( - dataset_properties: Optional[Dict[str, str]] = None, - standardize: Tuple[Tuple, bool] = ((True, False), True) + dataset_properties: Optional[Dict[str, str]] = None, + standardize: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='standardize', + value_range=(True, False), + default_value=True, + ), ) -> ConfigurationSpace: - standardize = CategoricalHyperparameter("standardize", - choices=standardize[0], - default_value=standardize[1]) - cs = ConfigurationSpace() - cs.add_hyperparameters([standardize]) + add_hyperparameter(cs, standardize, CategoricalHyperparameter) return cs diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/RandomKitchenSinks.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/RandomKitchenSinks.py index 8f03e1880..9dbf26cbc 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/RandomKitchenSinks.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/RandomKitchenSinks.py @@ -1,5 +1,5 @@ from math import ceil, floor -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, Optional, Union from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( @@ -14,6 +14,7 @@ from autoPyTorch.pipeline.components.preprocessing.tabular_preprocessing.feature_preprocessing. \ base_feature_preprocessor import autoPyTorchFeaturePreprocessingComponent +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter class RandomKitchenSinks(autoPyTorchFeaturePreprocessingComponent): @@ -35,35 +36,38 @@ def fit(self, X: Dict[str, Any], y: Any = None) -> BaseEstimator: @staticmethod def get_hyperparameter_search_space( dataset_properties: Optional[Dict[str, str]] = None, - n_components: Tuple[Tuple, float, bool] = ((0.5, 0.9), 0.5, True), - gamma: Tuple[Tuple, float, bool] = ((3.0517578125e-05, 8), 1.0, True), - degree: Tuple[Tuple, int] = ((2, 5), 3), - coef0: Tuple[Tuple, float] = ((-1, 1), 0) + n_components: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='n_components', + value_range=(0.5, 0.9), + default_value=0.5, + ), + gamma: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='gamma', + value_range=(3.0517578125e-05, 8), + default_value=1.0, + log=True), ) -> ConfigurationSpace: + cs = ConfigurationSpace() if dataset_properties is not None: n_features = len(dataset_properties['numerical_columns']) - # if numerical features are 1, set log to False if n_features == 1: log = False else: - log = n_components[2] - n_components = ((floor(n_components[0][0] * n_features), ceil(n_components[0][1] * n_features)), - ceil(n_components[1] * n_features), log) + log = n_components.log + n_components = HyperparameterSearchSpace(hyperparameter='n_components', + value_range=( + floor(float(n_components.value_range[0]) * n_features), + ceil(float(n_components.value_range[1]) * n_features)), + default_value=ceil(float(n_components.default_value) * n_features), + log=log) else: - n_components = ((10, 2000), 100, True) + n_components = HyperparameterSearchSpace(hyperparameter='n_components', + value_range=(10, 2000), + default_value=100, + log=n_components.log) - n_components = UniformIntegerHyperparameter( - "n_components", lower=n_components[0][0], upper=n_components[0][1], - default_value=n_components[1], log=n_components[2]) - gamma = UniformFloatHyperparameter( - "gamma", - lower=gamma[0][0], upper=gamma[0][1], - log=gamma[2], - default_value=gamma[1], - ) - cs = ConfigurationSpace() - cs.add_hyperparameters([n_components, gamma]) + add_hyperparameter(cs, n_components, UniformIntegerHyperparameter) + + add_hyperparameter(cs, gamma, UniformFloatHyperparameter) return cs @staticmethod diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/TruncatedSVD.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/TruncatedSVD.py index 558fdb4de..fdf6a751a 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/TruncatedSVD.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/TruncatedSVD.py @@ -1,5 +1,5 @@ from math import floor -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, Optional, Union from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( @@ -11,8 +11,9 @@ import sklearn.decomposition from sklearn.base import BaseEstimator -from autoPyTorch.pipeline.components.preprocessing.tabular_preprocessing.feature_preprocessing\ +from autoPyTorch.pipeline.components.preprocessing.tabular_preprocessing.feature_preprocessing \ .base_feature_preprocessor import autoPyTorchFeaturePreprocessingComponent +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter class TruncatedSVD(autoPyTorchFeaturePreprocessingComponent): @@ -38,18 +39,25 @@ def get_properties(dataset_properties: Optional[Dict[str, str]] = None) -> Dict[ @staticmethod def get_hyperparameter_search_space( dataset_properties: Optional[Dict[str, str]] = None, - target_dim: Tuple[Tuple, float] = ((0.5, 0.9), 0.5), + target_dim: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='target_dim', + value_range=(0.5, 0.9), + default_value=0.5, + ), ) -> ConfigurationSpace: cs = ConfigurationSpace() if dataset_properties is not None: n_features = len(dataset_properties['numerical_columns']) - target_dim = ((floor(target_dim[0][0] * n_features), floor(target_dim[0][1] * n_features)), - floor(target_dim[1] * n_features)) + target_dim = HyperparameterSearchSpace(hyperparameter=target_dim.hyperparameter, + value_range=(floor(float(target_dim.value_range[0]) * n_features), + floor(float(target_dim.value_range[1]) * n_features)), + default_value=floor(float(target_dim.default_value) * n_features), + log=target_dim.log) else: - target_dim = ((10, 256), 128) - target_dim = UniformIntegerHyperparameter("target_dim", lower=target_dim[0][0], - upper=target_dim[0][1], default_value=target_dim[1]) - cs.add_hyperparameters([target_dim]) + target_dim = HyperparameterSearchSpace(hyperparameter=target_dim.hyperparameter, + value_range=(10, 256), + default_value=128, + log=target_dim.log) + add_hyperparameter(cs, target_dim, UniformIntegerHyperparameter) return cs diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/base_feature_preprocessor_choice.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/base_feature_preprocessor_choice.py index 56af71877..43a1e1a66 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/base_feature_preprocessor_choice.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/base_feature_preprocessor_choice.py @@ -82,23 +82,39 @@ def get_hyperparameter_search_space(self, continue default = default_ break - - # add only no feature preprocessor to choice hyperparameters in case the dataset is only categorical - if len(dataset_properties['numerical_columns']) == 0: - default = 'NoFeaturePreprocessor' - if include is not None and default not in include: - raise ValueError("Provided {} in include, however, " - "the dataset is incompatible with it".format(include)) + updates = self._get_search_space_updates() + if '__choice__' in updates.keys(): + choice_hyperparameter = updates['__choice__'] + if not set(choice_hyperparameter.value_range).issubset(available_): + raise ValueError("Expected given update for {} to have " + "choices in {} got {}".format(self.__class__.__name__, + available_, + choice_hyperparameter.value_range)) + if len(dataset_properties['numerical_columns']) == 0: + assert len(choice_hyperparameter.value_range) == 1 + assert 'NoFeaturePreprocessor' in choice_hyperparameter.value_range, \ + "Provided {} in choices, however, the dataset " \ + "is incompatible with it".format(choice_hyperparameter.value_range) preprocessor = CSH.CategoricalHyperparameter('__choice__', - ['NoFeaturePreprocessor'], - default_value=default) + choice_hyperparameter.value_range, + default_value=choice_hyperparameter.default_value) else: - # Truncated SVD requires n_features > n_components - if len(dataset_properties['numerical_columns']) == 1: - del available_['TruncatedSVD'] - preprocessor = CSH.CategoricalHyperparameter('__choice__', - list(available_.keys()), - default_value=default) + # add only no feature preprocessor to choice hyperparameters in case the dataset is only categorical + if len(dataset_properties['numerical_columns']) == 0: + default = 'NoFeaturePreprocessor' + if include is not None and default not in include: + raise ValueError("Provided {} in include, however, " + "the dataset is incompatible with it".format(include)) + preprocessor = CSH.CategoricalHyperparameter('__choice__', + ['NoFeaturePreprocessor'], + default_value=default) + else: + # Truncated SVD requires n_features > n_components + if len(dataset_properties['numerical_columns']) == 1: + del available_['TruncatedSVD'] + preprocessor = CSH.CategoricalHyperparameter('__choice__', + list(available_.keys()), + default_value=default) cs.add_hyperparameter(preprocessor) diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/imputation/SimpleImputer.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/imputation/SimpleImputer.py index 6b2a81bc9..6f0c8a2c2 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/imputation/SimpleImputer.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/imputation/SimpleImputer.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, Optional, Union from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( @@ -10,6 +10,7 @@ from sklearn.impute import SimpleImputer as SklearnSimpleImputer from autoPyTorch.pipeline.components.preprocessing.tabular_preprocessing.imputation.base_imputer import BaseImputer +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter class SimpleImputer(BaseImputer): @@ -65,28 +66,29 @@ def fit(self, X: Dict[str, Any], y: Any = None) -> BaseImputer: return self @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict[str, Any]] = None, - numerical_strategy: Tuple[Tuple, str] = (("mean", "median", - "most_frequent", "constant_zero"), - "mean"), - categorical_strategy: Tuple[Tuple, str] = (("most_frequent", - "constant_!missing!"), - "most_frequent") - ) -> ConfigurationSpace: + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict[str, Any]] = None, + numerical_strategy: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='numerical_strategy', + value_range=("mean", "median", + "most_frequent", + "constant_zero"), + default_value="mean", + ), + categorical_strategy: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter='categorical_strategy', + value_range=("most_frequent", + "constant_!missing!"), + default_value="most_frequent") + ) -> ConfigurationSpace: cs = ConfigurationSpace() assert dataset_properties is not None, "To create hyperparameter search space" \ ", dataset_properties should not be None" if len(dataset_properties['numerical_columns']) != 0: - numerical_strategy = CategoricalHyperparameter("numerical_strategy", - numerical_strategy[0], - default_value=numerical_strategy[1]) - cs.add_hyperparameter(numerical_strategy) + add_hyperparameter(cs, numerical_strategy, CategoricalHyperparameter) if len(dataset_properties['categorical_columns']) != 0: - categorical_strategy = CategoricalHyperparameter("categorical_strategy", - categorical_strategy[0], - default_value=categorical_strategy[1]) - cs.add_hyperparameter(categorical_strategy) + add_hyperparameter(cs, categorical_strategy, CategoricalHyperparameter) + return cs @staticmethod diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/Normalizer.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/Normalizer.py index f0e6dc0ff..374188812 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/Normalizer.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/Normalizer.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, Optional, Union from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( @@ -10,6 +10,7 @@ from sklearn.preprocessing import Normalizer as SklearnNormalizer from autoPyTorch.pipeline.components.preprocessing.tabular_preprocessing.scaling.base_scaler import BaseScaler +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter class Normalizer(BaseScaler): @@ -36,13 +37,15 @@ def fit(self, X: Dict[str, Any], y: Any = None) -> BaseScaler: return self @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict[str, Any]] = None, - norm: Tuple[Tuple, str] = (("mean_abs", "mean_squared", "max"), - "mean_squared") - ) -> ConfigurationSpace: + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict[str, Any]] = None, + norm: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="norm", + value_range=("mean_abs", "mean_squared", "max"), + default_value="mean_squared", + ) + ) -> ConfigurationSpace: cs = ConfigurationSpace() - norm = CategoricalHyperparameter("norm", norm[0], default_value=norm[1]) - cs.add_hyperparameter(norm) + add_hyperparameter(cs, norm, CategoricalHyperparameter) return cs @staticmethod diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/base_scaler_choice.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/base_scaler_choice.py index 6fdcc47bc..7c5f22fd5 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/base_scaler_choice.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/base_scaler_choice.py @@ -70,20 +70,37 @@ def get_hyperparameter_search_space(self, if default_ in available_scalers: default = default_ break + updates = self._get_search_space_updates() + if '__choice__' in updates.keys(): + choice_hyperparameter = updates['__choice__'] + if not set(choice_hyperparameter.value_range).issubset(available_scalers): + raise ValueError("Expected given update for {} to have " + "choices in {} got {}".format(self.__class__.__name__, + available_scalers, + choice_hyperparameter.value_range)) + if len(dataset_properties['numerical_columns']) == 0: + assert len(choice_hyperparameter.value_range) == 1 + if 'NoScaler' not in choice_hyperparameter.value_range: + raise ValueError("Provided {} in choices, however, the dataset " + "is incompatible with it".format(choice_hyperparameter.value_range)) - # add only no scaler to choice hyperparameters in case the dataset is only categorical - if len(dataset_properties['numerical_columns']) == 0: - default = 'NoScaler' - if include is not None and default not in include: - raise ValueError("Provided {} in include, however, " - "the dataset is incompatible with it".format(include)) preprocessor = CSH.CategoricalHyperparameter('__choice__', - ['NoScaler'], - default_value=default) + choice_hyperparameter.value_range, + default_value=choice_hyperparameter.default_value) else: - preprocessor = CSH.CategoricalHyperparameter('__choice__', - list(available_scalers.keys()), - default_value=default) + # add only no scaler to choice hyperparameters in case the dataset is only categorical + if len(dataset_properties['numerical_columns']) == 0: + default = 'NoScaler' + if include is not None and default not in include: + raise ValueError("Provided {} in include, however, " + "the dataset is incompatible with it".format(include)) + preprocessor = CSH.CategoricalHyperparameter('__choice__', + ['NoScaler'], + default_value=default) + else: + preprocessor = CSH.CategoricalHyperparameter('__choice__', + list(available_scalers.keys()), + default_value=default) cs.add_hyperparameter(preprocessor) # add only child hyperparameters of early_preprocessor choices diff --git a/autoPyTorch/pipeline/components/setup/augmentation/image/GaussianBlur.py b/autoPyTorch/pipeline/components/setup/augmentation/image/GaussianBlur.py index e977b0633..8fca40999 100644 --- a/autoPyTorch/pipeline/components/setup/augmentation/image/GaussianBlur.py +++ b/autoPyTorch/pipeline/components/setup/augmentation/image/GaussianBlur.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, Optional, Union import ConfigSpace as CS from ConfigSpace.configuration_space import ConfigurationSpace @@ -13,6 +13,7 @@ import numpy as np from autoPyTorch.pipeline.components.setup.augmentation.image.base_image_augmenter import BaseImageAugmenter +from autoPyTorch.utils.common import HyperparameterSearchSpace, get_hyperparameter class GaussianBlur(BaseImageAugmenter): @@ -31,20 +32,24 @@ def fit(self, X: Dict[str, Any], y: Any = None) -> BaseImageAugmenter: @staticmethod def get_hyperparameter_search_space( dataset_properties: Optional[Dict[str, str]] = None, - use_augmenter: Tuple[Tuple, bool] = ((True, False), True), - sigma_min: Tuple[Tuple, int] = ((0, 3), 0), - sigma_offset: Tuple[Tuple, float] = ((0.0, 3.0), 0.5), + use_augmenter: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_augmenter", + value_range=(True, False), + default_value=True, + ), + sigma_min: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="sigma_min", + value_range=(0, 3), + default_value=0, + ), + sigma_offset: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="sigma_offset", + value_range=(0.0, 3.0), + default_value=0.5, + ), ) -> ConfigurationSpace: - cs = ConfigurationSpace() - use_augmenter = CategoricalHyperparameter('use_augmenter', choices=use_augmenter[0], - default_value=use_augmenter[1]) - sigma_min = UniformFloatHyperparameter('sigma_min', lower=sigma_min[0][0], upper=sigma_min[0][1], - default_value=0) - sigma_offset = UniformFloatHyperparameter('sigma_offset', lower=sigma_offset[0][0], upper=sigma_offset[0][1], - default_value=0.5) + use_augmenter = get_hyperparameter(use_augmenter, CategoricalHyperparameter) + sigma_min = get_hyperparameter(sigma_min, UniformFloatHyperparameter) + sigma_offset = get_hyperparameter(sigma_offset, UniformFloatHyperparameter) cs.add_hyperparameters([use_augmenter, sigma_min, sigma_offset]) - # only add hyperparameters to configuration space if we are using the augmenter cs.add_condition(CS.EqualsCondition(sigma_min, use_augmenter, True)) cs.add_condition(CS.EqualsCondition(sigma_offset, use_augmenter, True)) diff --git a/autoPyTorch/pipeline/components/setup/augmentation/image/GaussianNoise.py b/autoPyTorch/pipeline/components/setup/augmentation/image/GaussianNoise.py index 064c564cd..48178432e 100644 --- a/autoPyTorch/pipeline/components/setup/augmentation/image/GaussianNoise.py +++ b/autoPyTorch/pipeline/components/setup/augmentation/image/GaussianNoise.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, Optional, Union import ConfigSpace as CS from ConfigSpace.configuration_space import ConfigurationSpace @@ -13,6 +13,7 @@ import numpy as np from autoPyTorch.pipeline.components.setup.augmentation.image.base_image_augmenter import BaseImageAugmenter +from autoPyTorch.utils.common import HyperparameterSearchSpace, get_hyperparameter class GaussianNoise(BaseImageAugmenter): @@ -30,15 +31,19 @@ def fit(self, X: Dict[str, Any], y: Any = None) -> BaseImageAugmenter: @staticmethod def get_hyperparameter_search_space( dataset_properties: Optional[Dict[str, str]] = None, - use_augmenter: Tuple[Tuple[bool, bool], bool] = ((True, False), True), - sigma_offset: Tuple[Tuple[float, float], float] = ((0.0, 3.0), 0.3) + use_augmenter: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_augmenter", + value_range=(True, False), + default_value=True, + ), + sigma_offset: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="sigma_offset", + value_range=(0.0, 3.0), + default_value=0.3, + ), ) -> ConfigurationSpace: cs = ConfigurationSpace() - sigma_offset = UniformFloatHyperparameter('sigma_offset', lower=sigma_offset[0][0], upper=sigma_offset[0][1], - default_value=sigma_offset[1]) - use_augmenter = CategoricalHyperparameter('use_augmenter', choices=use_augmenter[0], - default_value=use_augmenter[1]) + use_augmenter = get_hyperparameter(use_augmenter, CategoricalHyperparameter) + sigma_offset = get_hyperparameter(sigma_offset, UniformFloatHyperparameter) cs.add_hyperparameters([use_augmenter, sigma_offset]) # only add hyperparameters to configuration space if we are using the augmenter cs.add_condition(CS.EqualsCondition(sigma_offset, use_augmenter, True)) diff --git a/autoPyTorch/pipeline/components/setup/augmentation/image/RandomAffine.py b/autoPyTorch/pipeline/components/setup/augmentation/image/RandomAffine.py index a9797a6e5..3db351ee2 100644 --- a/autoPyTorch/pipeline/components/setup/augmentation/image/RandomAffine.py +++ b/autoPyTorch/pipeline/components/setup/augmentation/image/RandomAffine.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, Optional, Union import ConfigSpace as CS from ConfigSpace.configuration_space import ConfigurationSpace @@ -14,6 +14,7 @@ import numpy as np from autoPyTorch.pipeline.components.setup.augmentation.image.base_image_augmenter import BaseImageAugmenter +from autoPyTorch.utils.common import HyperparameterSearchSpace, get_hyperparameter class RandomAffine(BaseImageAugmenter): @@ -38,29 +39,36 @@ def fit(self, X: Dict[str, Any], y: Any = None) -> BaseImageAugmenter: @staticmethod def get_hyperparameter_search_space( dataset_properties: Optional[Dict[str, str]] = None, - use_augmenter: Tuple[Tuple, bool] = ((True, False), True), - scale_offset: Tuple[Tuple, float] = ((0, 0.4), 0.2), - translate_percent_offset: Tuple[Tuple, float] = ((0, 0.4), 0.2), - shear: Tuple[Tuple, int] = ((0, 45), 30), - rotate: Tuple[Tuple, int] = ((0, 360), 45) + use_augmenter: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_augmenter", + value_range=(True, False), + default_value=True, + ), + scale_offset: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="scale_offset", + value_range=(0, 0.4), + default_value=0.2, + ), + translate_percent_offset: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="translate_percent_offset", + value_range=(0, 0.4), + default_value=0.2), + shear: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="shear", + value_range=(0, 45), + default_value=30, + ), + rotate: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="rotate", + value_range=(0, 360), + default_value=45, + ), ) -> ConfigurationSpace: - cs = ConfigurationSpace() - scale_offset = UniformFloatHyperparameter('scale_offset', lower=scale_offset[0][0], upper=scale_offset[0][1], - default_value=scale_offset[1]) - - translate_percent_offset = UniformFloatHyperparameter('translate_percent_offset', - lower=translate_percent_offset[0][0], - upper=translate_percent_offset[0][1], - default_value=translate_percent_offset[1]) - shear = UniformIntegerHyperparameter('shear', lower=shear[0][0], upper=shear[0][1], default_value=shear[1]) - rotate = UniformIntegerHyperparameter('rotate', lower=0, upper=360, default_value=45) - - use_augmenter = CategoricalHyperparameter('use_augmenter', choices=use_augmenter[0], - default_value=use_augmenter[1]) - cs.add_hyperparameters([scale_offset, translate_percent_offset]) - cs.add_hyperparameters([shear, rotate, use_augmenter]) + use_augmenter = get_hyperparameter(use_augmenter, CategoricalHyperparameter) + scale_offset = get_hyperparameter(scale_offset, UniformFloatHyperparameter) + translate_percent_offset = get_hyperparameter(translate_percent_offset, UniformFloatHyperparameter) + shear = get_hyperparameter(shear, UniformIntegerHyperparameter) + rotate = get_hyperparameter(rotate, UniformIntegerHyperparameter) + cs.add_hyperparameters([use_augmenter, scale_offset, translate_percent_offset]) + cs.add_hyperparameters([shear, rotate]) # only add hyperparameters to configuration space if we are using the augmenter cs.add_condition(CS.EqualsCondition(scale_offset, use_augmenter, True)) diff --git a/autoPyTorch/pipeline/components/setup/augmentation/image/RandomCutout.py b/autoPyTorch/pipeline/components/setup/augmentation/image/RandomCutout.py index 332efbb41..493375ca7 100644 --- a/autoPyTorch/pipeline/components/setup/augmentation/image/RandomCutout.py +++ b/autoPyTorch/pipeline/components/setup/augmentation/image/RandomCutout.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, Optional, Union import ConfigSpace as CS from ConfigSpace.configuration_space import ConfigurationSpace @@ -13,6 +13,7 @@ import numpy as np from autoPyTorch.pipeline.components.setup.augmentation.image.base_image_augmenter import BaseImageAugmenter +from autoPyTorch.utils.common import HyperparameterSearchSpace, get_hyperparameter class RandomCutout(BaseImageAugmenter): @@ -31,16 +32,21 @@ def fit(self, X: Dict[str, Any], y: Any = None) -> BaseImageAugmenter: @staticmethod def get_hyperparameter_search_space( - dataset_properties: Optional[Dict[str, str]] = None, - use_augmenter: Tuple[Tuple, bool] = ((True, False), True), - p: Tuple[Tuple, float] = ((0.2, 1.0), 0.5) + dataset_properties: Optional[Dict[str, str]] = None, + use_augmenter: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_augmenter", + value_range=(True, False), + default_value=True, + ), + p: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="p", + value_range=(0.2, 1.0), + default_value=0.5, + ), ) -> ConfigurationSpace: cs = ConfigurationSpace() - p = UniformFloatHyperparameter('p', lower=p[0][0], upper=p[0][1], default_value=p[1]) - use_augmenter = CategoricalHyperparameter('use_augmenter', choices=use_augmenter[0], - default_value=use_augmenter[1]) - cs.add_hyperparameters([p, use_augmenter]) + use_augmenter = get_hyperparameter(use_augmenter, CategoricalHyperparameter) + p = get_hyperparameter(p, UniformFloatHyperparameter) + cs.add_hyperparameters([use_augmenter, p]) # only add hyperparameters to configuration space if we are using the augmenter cs.add_condition(CS.EqualsCondition(p, use_augmenter, True)) return cs diff --git a/autoPyTorch/pipeline/components/setup/augmentation/image/Resize.py b/autoPyTorch/pipeline/components/setup/augmentation/image/Resize.py index eabafa95b..14001a8a7 100644 --- a/autoPyTorch/pipeline/components/setup/augmentation/image/Resize.py +++ b/autoPyTorch/pipeline/components/setup/augmentation/image/Resize.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, Optional, Union from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( @@ -11,7 +11,7 @@ import numpy as np from autoPyTorch.pipeline.components.setup.augmentation.image.base_image_augmenter import BaseImageAugmenter -from autoPyTorch.utils.common import FitRequirement +from autoPyTorch.utils.common import FitRequirement, HyperparameterSearchSpace, add_hyperparameter class Resize(BaseImageAugmenter): @@ -35,13 +35,14 @@ def fit(self, X: Dict[str, Any], y: Any = None) -> BaseImageAugmenter: @staticmethod def get_hyperparameter_search_space( - dataset_properties: Optional[Dict[str, str]] = None, - use_augmenter: Tuple[Tuple, bool] = ((True, False), True), + dataset_properties: Optional[Dict[str, str]] = None, + use_augmenter: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_augmenter", + value_range=(True, False), + default_value=True, + ), ) -> ConfigurationSpace: cs = ConfigurationSpace() - use_augmenter = CategoricalHyperparameter('use_augmenter', choices=use_augmenter[0], - default_value=use_augmenter[1]) - cs.add_hyperparameters([use_augmenter]) + add_hyperparameter(cs, use_augmenter, CategoricalHyperparameter) return cs diff --git a/autoPyTorch/pipeline/components/setup/augmentation/image/ZeroPadAndCrop.py b/autoPyTorch/pipeline/components/setup/augmentation/image/ZeroPadAndCrop.py index 257c0a550..df18e48b4 100644 --- a/autoPyTorch/pipeline/components/setup/augmentation/image/ZeroPadAndCrop.py +++ b/autoPyTorch/pipeline/components/setup/augmentation/image/ZeroPadAndCrop.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, Optional, Union from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( @@ -11,7 +11,7 @@ import numpy as np from autoPyTorch.pipeline.components.setup.augmentation.image.base_image_augmenter import BaseImageAugmenter -from autoPyTorch.utils.common import FitRequirement +from autoPyTorch.utils.common import FitRequirement, HyperparameterSearchSpace, add_hyperparameter class ZeroPadAndCrop(BaseImageAugmenter): @@ -42,13 +42,14 @@ def fit(self, X: Dict[str, Any], y: Any = None) -> BaseImageAugmenter: @staticmethod def get_hyperparameter_search_space( dataset_properties: Optional[Dict[str, str]] = None, - percent: Tuple[Tuple, float] = ((0, 0.5), 0.1) + percent: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='percent', + value_range=(0, 0.5), + default_value=0.1, + ) ) -> ConfigurationSpace: cs = ConfigurationSpace() - percent = UniformFloatHyperparameter('percent', lower=percent[0][0], upper=percent[0][1], - default_value=percent[1]) - cs.add_hyperparameters([percent]) + add_hyperparameter(cs, percent, UniformFloatHyperparameter) return cs @staticmethod diff --git a/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingLR.py b/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingLR.py index 9cbbaa41d..05351f3aa 100644 --- a/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingLR.py +++ b/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingLR.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple +from typing import Any, Dict, Optional from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( @@ -11,6 +11,7 @@ from torch.optim.lr_scheduler import _LRScheduler from autoPyTorch.pipeline.components.setup.lr_scheduler.base_scheduler import BaseLRComponent +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter class CosineAnnealingLR(BaseLRComponent): @@ -56,16 +57,18 @@ def fit(self, X: Dict[str, Any], y: Any = None) -> BaseLRComponent: @staticmethod def get_properties(dataset_properties: Optional[Dict[str, Any]] = None) -> Dict[str, str]: return { - 'shortname': 'CosineAnnealingWarmRestarts', - 'name': 'Cosine Annealing WarmRestarts', + 'shortname': 'CosineAnnealing', + 'name': 'Cosine Annealing', } @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, - T_max: Tuple[Tuple[int, int], int] = ((10, 500), 200) - ) -> ConfigurationSpace: - T_max = UniformIntegerHyperparameter( - "T_max", T_max[0][0], T_max[0][1], default_value=T_max[1]) + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict] = None, + T_max: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='T_max', + value_range=(10, 500), + default_value=200, + ) + ) -> ConfigurationSpace: cs = ConfigurationSpace() - cs.add_hyperparameters([T_max]) + add_hyperparameter(cs, T_max, UniformIntegerHyperparameter) return cs diff --git a/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingWarmRestarts.py b/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingWarmRestarts.py index 07c9ffc2a..6e64af802 100644 --- a/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingWarmRestarts.py +++ b/autoPyTorch/pipeline/components/setup/lr_scheduler/CosineAnnealingWarmRestarts.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple +from typing import Any, Dict, Optional from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import UniformFloatHyperparameter, UniformIntegerHyperparameter @@ -9,6 +9,7 @@ from torch.optim.lr_scheduler import _LRScheduler from autoPyTorch.pipeline.components.setup.lr_scheduler.base_scheduler import BaseLRComponent +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter class CosineAnnealingWarmRestarts(BaseLRComponent): @@ -30,7 +31,6 @@ def __init__( T_mult: int, random_state: Optional[np.random.RandomState] = None ): - super().__init__() self.T_0 = T_0 self.T_mult = T_mult @@ -67,14 +67,19 @@ def get_properties(dataset_properties: Optional[Dict[str, Any]] = None) -> Dict[ } @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, - T_0: Tuple[Tuple[int, int], int] = ((1, 20), 1), - T_mult: Tuple[Tuple[float, float], float] = ((1.0, 2.0), 1.0) - ) -> ConfigurationSpace: - T_0 = UniformIntegerHyperparameter( - "T_0", T_0[0][0], T_0[0][1], default_value=T_0[1]) - T_mult = UniformFloatHyperparameter( - "T_mult", T_mult[0][0], T_mult[0][1], default_value=T_mult[1]) + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict] = None, + T_0: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='T_0', + value_range=(1, 20), + default_value=1, + ), + T_mult: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='T_mult', + value_range=(1.0, 2.0), + default_value=1.0, + ), + ) -> ConfigurationSpace: cs = ConfigurationSpace() - cs.add_hyperparameters([T_0, T_mult]) + add_hyperparameter(cs, T_0, UniformIntegerHyperparameter) + add_hyperparameter(cs, T_mult, UniformFloatHyperparameter) + return cs diff --git a/autoPyTorch/pipeline/components/setup/lr_scheduler/CyclicLR.py b/autoPyTorch/pipeline/components/setup/lr_scheduler/CyclicLR.py index bc6e4e3ff..972134064 100644 --- a/autoPyTorch/pipeline/components/setup/lr_scheduler/CyclicLR.py +++ b/autoPyTorch/pipeline/components/setup/lr_scheduler/CyclicLR.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple +from typing import Any, Dict, Optional from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( @@ -13,6 +13,7 @@ from torch.optim.lr_scheduler import _LRScheduler from autoPyTorch.pipeline.components.setup.lr_scheduler.base_scheduler import BaseLRComponent +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter class CyclicLR(BaseLRComponent): @@ -31,6 +32,7 @@ class CyclicLR(BaseLRComponent): base_lr. This simplifies the learning space """ + def __init__( self, base_lr: float, @@ -39,7 +41,6 @@ def __init__( max_lr: float = 0.1, random_state: Optional[np.random.RandomState] = None ): - super().__init__() self.base_lr = base_lr self.mode = mode @@ -82,24 +83,36 @@ def fit(self, X: Dict[str, Any], y: Any = None) -> BaseLRComponent: def get_properties(dataset_properties: Optional[Dict[str, Any]] = None) -> Dict[str, str]: return { 'shortname': 'CyclicLR', - 'name': 'CyclicLR', + 'name': 'Cyclic Learning Rate Scheduler', } @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, - base_lr: Tuple[Tuple, float] = ((1e-6, 1e-1), 0.01), - mode: Tuple[Tuple, str] = (('triangular', 'triangular2', 'exp_range'), - 'triangular'), - step_size_up: Tuple[Tuple, int] = ((1000, 4000), 2000), - max_lr: Tuple[Tuple, float] = ((1e-3, 1e-1), 0.1) - ) -> ConfigurationSpace: - base_lr = UniformFloatHyperparameter( - "base_lr", base_lr[0][0], base_lr[0][1], default_value=base_lr[1]) - mode = CategoricalHyperparameter('mode', choices=mode[0], default_value=mode[1]) - step_size_up = UniformIntegerHyperparameter( - "step_size_up", step_size_up[0][0], step_size_up[0][1], default_value=step_size_up[1]) - max_lr = UniformFloatHyperparameter( - "max_lr", max_lr[0][0], max_lr[0][1], default_value=max_lr[1]) + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict] = None, + base_lr: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='base_lr', + value_range=(1e-6, 1e-1), + default_value=0.01, + ), + mode: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='mode', + value_range=('triangular', + 'triangular2', + 'exp_range'), + default_value='triangular', + ), + step_size_up: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='step_size_up', + value_range=(1000, 4000), + default_value=2000, + ), + max_lr: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='max_lr', + value_range=(1e-3, 1e-1), + default_value=0.1, + ) + ) -> ConfigurationSpace: cs = ConfigurationSpace() - cs.add_hyperparameters([base_lr, mode, step_size_up, max_lr]) + + add_hyperparameter(cs, base_lr, UniformFloatHyperparameter) + add_hyperparameter(cs, mode, CategoricalHyperparameter) + add_hyperparameter(cs, step_size_up, UniformIntegerHyperparameter) + add_hyperparameter(cs, max_lr, UniformFloatHyperparameter) + return cs diff --git a/autoPyTorch/pipeline/components/setup/lr_scheduler/ExponentialLR.py b/autoPyTorch/pipeline/components/setup/lr_scheduler/ExponentialLR.py index d090d710b..177c9d551 100644 --- a/autoPyTorch/pipeline/components/setup/lr_scheduler/ExponentialLR.py +++ b/autoPyTorch/pipeline/components/setup/lr_scheduler/ExponentialLR.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple +from typing import Any, Dict, Optional from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( @@ -11,6 +11,7 @@ from torch.optim.lr_scheduler import _LRScheduler from autoPyTorch.pipeline.components.setup.lr_scheduler.base_scheduler import BaseLRComponent +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter class ExponentialLR(BaseLRComponent): @@ -58,15 +59,17 @@ def fit(self, X: Dict[str, Any], y: Any = None) -> BaseLRComponent: def get_properties(dataset_properties: Optional[Dict[str, Any]] = None) -> Dict[str, str]: return { 'shortname': 'ExponentialLR', - 'name': 'ExponentialLR', + 'name': 'Exponential Learning Rate Scheduler', } @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, - gamma: Tuple[Tuple, float] = ((0.7, 0.9999), 0.9) - ) -> ConfigurationSpace: - gamma = UniformFloatHyperparameter( - "gamma", gamma[0][0], gamma[0][1], default_value=gamma[1]) + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict] = None, + gamma: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='gamma', + value_range=(0.7, 0.9999), + default_value=0.9, + ) + ) -> ConfigurationSpace: cs = ConfigurationSpace() - cs.add_hyperparameters([gamma]) + add_hyperparameter(cs, gamma, UniformFloatHyperparameter) return cs diff --git a/autoPyTorch/pipeline/components/setup/lr_scheduler/ReduceLROnPlateau.py b/autoPyTorch/pipeline/components/setup/lr_scheduler/ReduceLROnPlateau.py index bd8c9c97a..60711b387 100644 --- a/autoPyTorch/pipeline/components/setup/lr_scheduler/ReduceLROnPlateau.py +++ b/autoPyTorch/pipeline/components/setup/lr_scheduler/ReduceLROnPlateau.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple +from typing import Any, Dict, Optional from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( @@ -13,6 +13,7 @@ from torch.optim.lr_scheduler import _LRScheduler from autoPyTorch.pipeline.components.setup.lr_scheduler.base_scheduler import BaseLRComponent +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter class ReduceLROnPlateau(BaseLRComponent): @@ -31,6 +32,7 @@ class ReduceLROnPlateau(BaseLRComponent): rate will be reduced. random_state (Optional[np.random.RandomState]): random state """ + def __init__( self, mode: str, @@ -38,7 +40,6 @@ def __init__( patience: int, random_state: Optional[np.random.RandomState] = None ): - super().__init__() self.mode = mode self.factor = factor @@ -77,16 +78,26 @@ def get_properties(dataset_properties: Optional[Dict[str, Any]] = None) -> Dict[ } @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, - mode: Tuple[Tuple, str] = (('min', 'max'), 'min'), - patience: Tuple[Tuple, int] = ((5, 20), 10), - factor: Tuple[Tuple[float, float], float] = ((0.01, 0.9), 0.1) - ) -> ConfigurationSpace: - mode = CategoricalHyperparameter('mode', choices=mode[0], default_value=mode[1]) - patience = UniformIntegerHyperparameter( - "patience", patience[0][0], patience[0][1], default_value=patience[1]) - factor = UniformFloatHyperparameter( - "factor", factor[0][0], factor[0][1], default_value=factor[1]) + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict] = None, + mode: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='mode', + value_range=('min', 'max'), + default_value='min', + ), + patience: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='patience', + value_range=(5, 20), + default_value=10, + ), + factor: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='factor', + value_range=(0.01, 0.9), + default_value=0.1, + ), + ) -> ConfigurationSpace: + cs = ConfigurationSpace() - cs.add_hyperparameters([mode, patience, factor]) + + add_hyperparameter(cs, mode, CategoricalHyperparameter) + add_hyperparameter(cs, patience, UniformIntegerHyperparameter) + add_hyperparameter(cs, factor, UniformFloatHyperparameter) + return cs diff --git a/autoPyTorch/pipeline/components/setup/lr_scheduler/StepLR.py b/autoPyTorch/pipeline/components/setup/lr_scheduler/StepLR.py index 035e4e841..9f3b97db8 100644 --- a/autoPyTorch/pipeline/components/setup/lr_scheduler/StepLR.py +++ b/autoPyTorch/pipeline/components/setup/lr_scheduler/StepLR.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple +from typing import Any, Dict, Optional from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( @@ -12,6 +12,7 @@ from torch.optim.lr_scheduler import _LRScheduler from autoPyTorch.pipeline.components.setup.lr_scheduler.base_scheduler import BaseLRComponent +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter class StepLR(BaseLRComponent): @@ -25,13 +26,13 @@ class StepLR(BaseLRComponent): gamma (float) – Multiplicative factor of learning rate decay. Default: 0.1. """ + def __init__( self, step_size: int, gamma: float, random_state: Optional[np.random.RandomState] = None ): - super().__init__() self.gamma = gamma self.step_size = step_size @@ -68,14 +69,20 @@ def get_properties(dataset_properties: Optional[Dict[str, Any]] = None) -> Dict[ } @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, - gamma: Tuple[Tuple, float] = ((0.001, 0.9), 0.1), - step_size: Tuple[Tuple, int] = ((1, 10), 5) - ) -> ConfigurationSpace: - gamma = UniformFloatHyperparameter( - "gamma", gamma[0][0], gamma[0][1], default_value=gamma[1]) - step_size = UniformIntegerHyperparameter( - "step_size", step_size[0][0], step_size[0][1], default_value=step_size[1]) + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict] = None, + gamma: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='gamma', + value_range=(0.001, 0.9), + default_value=0.1, + ), + step_size: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='step_size', + value_range=(1, 10), + default_value=5, + ), + ) -> ConfigurationSpace: cs = ConfigurationSpace() - cs.add_hyperparameters([gamma, step_size]) + + add_hyperparameter(cs, step_size, UniformIntegerHyperparameter) + add_hyperparameter(cs, gamma, UniformFloatHyperparameter) + return cs diff --git a/autoPyTorch/pipeline/components/setup/lr_scheduler/base_scheduler_choice.py b/autoPyTorch/pipeline/components/setup/lr_scheduler/base_scheduler_choice.py index 0be6aec96..9349ab642 100644 --- a/autoPyTorch/pipeline/components/setup/lr_scheduler/base_scheduler_choice.py +++ b/autoPyTorch/pipeline/components/setup/lr_scheduler/base_scheduler_choice.py @@ -15,7 +15,6 @@ ) from autoPyTorch.pipeline.components.setup.lr_scheduler.base_scheduler import BaseLRComponent - directory = os.path.split(__file__)[0] _schedulers = find_components(__package__, directory, @@ -150,14 +149,25 @@ def get_hyperparameter_search_space( if default_ in available_schedulers: default = default_ break - - scheduler = CSH.CategoricalHyperparameter( - '__choice__', - list(available_schedulers.keys()), - default_value=default - ) + updates = self._get_search_space_updates() + if '__choice__' in updates.keys(): + choice_hyperparameter = updates['__choice__'] + if not set(choice_hyperparameter.value_range).issubset(available_schedulers): + raise ValueError("Expected given update for {} to have " + "choices in {} got {}".format(self.__class__.__name__, + available_schedulers, + choice_hyperparameter.value_range)) + scheduler = CSH.CategoricalHyperparameter('__choice__', + choice_hyperparameter.value_range, + default_value=choice_hyperparameter.default_value) + else: + scheduler = CSH.CategoricalHyperparameter( + '__choice__', + list(available_schedulers.keys()), + default_value=default + ) cs.add_hyperparameter(scheduler) - for name in available_schedulers: + for name in scheduler.choices: updates = self._get_search_space_updates(prefix=name) config_space = available_schedulers[name].get_hyperparameter_search_space(dataset_properties, # type:ignore **updates) diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/ConvNetImageBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/ConvNetImageBackbone.py index a9d1855c8..162e9f4e6 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/ConvNetImageBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/ConvNetImageBackbone.py @@ -11,6 +11,7 @@ from autoPyTorch.pipeline.components.setup.network_backbone.base_network_backbone import NetworkBackboneComponent from autoPyTorch.pipeline.components.setup.network_backbone.utils import _activations +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter class ConvNetImageBackbone(NetworkBackboneComponent): @@ -67,56 +68,45 @@ def get_properties(dataset_properties: Optional[Dict[str, str]] = None) -> Dict[ } @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, - num_layers: Tuple[Tuple, int] = ((2, 8), 4), - num_init_filters: Tuple[Tuple, int] = ((16, 64), 32), - activation: Tuple[Tuple, str] = (tuple(_activations.keys()), - list(_activations.keys())[0]), - kernel_size: Tuple[Tuple, int] = ((3, 5), 3), - stride: Tuple[Tuple, int] = ((1, 3), 1), - padding: Tuple[Tuple, int] = ((2, 3), 2), - pool_size: Tuple[Tuple, int] = ((2, 3), 2) - ) -> ConfigurationSpace: + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict] = None, + num_layers: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='num_layers', + value_range=(2, 8), + default_value=4, + ), + conv_init_filters: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='conv_init_filters', + value_range=(16, 64), + default_value=32, + ), + activation: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='activation', + value_range=tuple(_activations.keys()), + default_value=list(_activations.keys())[0], + ), + conv_kernel_size: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='conv_kernel_size', + value_range=(3, 5), + default_value=3, + ), + conv_kernel_stride: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='conv_kernel_stride', + value_range=(1, 3), + default_value=1, + ), + conv_kernel_padding: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='conv_kernel_padding', + value_range=(2, 3), + default_value=2, + ), + pool_size: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='pool_size', + value_range=(2, 3), + default_value=2, + ) + ) -> ConfigurationSpace: cs = CS.ConfigurationSpace() - min_num_layers, max_num_layers = num_layers[0] - cs.add_hyperparameter(UniformIntegerHyperparameter('num_layers', - lower=min_num_layers, - upper=max_num_layers, - default_value=num_layers[1])) - - cs.add_hyperparameter(CategoricalHyperparameter('activation', - choices=activation[0], - default_value=activation[1])) - - min_init_filters, max_init_filters = num_init_filters[0] - cs.add_hyperparameter(UniformIntegerHyperparameter('conv_init_filters', - lower=min_init_filters, - upper=max_init_filters, - default_value=num_init_filters[1])) - - min_kernel_size, max_kernel_size = kernel_size[0] - cs.add_hyperparameter(UniformIntegerHyperparameter('conv_kernel_size', - lower=min_kernel_size, - upper=max_kernel_size, - default_value=kernel_size[1])) - - min_stride, max_stride = stride[0] - cs.add_hyperparameter(UniformIntegerHyperparameter('conv_kernel_stride', - lower=min_stride, - upper=max_stride, - default_value=stride[1])) - - min_padding, max_padding = padding[0] - cs.add_hyperparameter(UniformIntegerHyperparameter('conv_kernel_padding', - lower=min_padding, - upper=max_padding, - default_value=padding[1])) - - min_pool_size, max_pool_size = pool_size[0] - cs.add_hyperparameter(UniformIntegerHyperparameter('pool_size', - lower=min_pool_size, - upper=max_pool_size, - default_value=pool_size[1])) + add_hyperparameter(cs, num_layers, UniformIntegerHyperparameter) + add_hyperparameter(cs, conv_init_filters, UniformIntegerHyperparameter) + add_hyperparameter(cs, activation, CategoricalHyperparameter) + add_hyperparameter(cs, conv_kernel_size, UniformIntegerHyperparameter) + add_hyperparameter(cs, conv_kernel_stride, UniformIntegerHyperparameter) + add_hyperparameter(cs, conv_kernel_padding, UniformIntegerHyperparameter) + add_hyperparameter(cs, pool_size, UniformIntegerHyperparameter) return cs diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/DenseNetImageBackone.py b/autoPyTorch/pipeline/components/setup/network_backbone/DenseNetImageBackbone.py similarity index 66% rename from autoPyTorch/pipeline/components/setup/network_backbone/DenseNetImageBackone.py rename to autoPyTorch/pipeline/components/setup/network_backbone/DenseNetImageBackbone.py index 98e0eb9b8..f32cd1640 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/DenseNetImageBackone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/DenseNetImageBackbone.py @@ -16,6 +16,7 @@ from autoPyTorch.pipeline.components.setup.network_backbone.base_network_backbone import NetworkBackboneComponent from autoPyTorch.pipeline.components.setup.network_backbone.utils import _activations +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter class _DenseLayer(nn.Sequential): @@ -92,7 +93,7 @@ def build_backbone(self, input_shape: Tuple[int, ...]) -> nn.Module: channels, iw, ih = input_shape growth_rate = self.config['growth_rate'] - block_config = [self.config['layer_in_block_%d' % (i + 1)] for i in range(self.config['blocks'])] + block_config = [self.config['layer_in_block_%d' % (i + 1)] for i in range(self.config['num_blocks'])] num_init_features = 2 * growth_rate bn_size = 4 drop_rate = self.config['dropout'] if self.config['use_dropout'] else 0 @@ -152,58 +153,61 @@ def get_properties(dataset_properties: Optional[Dict[str, str]] = None) -> Dict[ } @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, - num_blocks: Tuple[Tuple, int] = ((3, 4), 3), - num_layers: Tuple[Tuple, int] = ((4, 64), 16), - growth_rate: Tuple[Tuple, int] = ((12, 40), 20), - activation: Tuple[Tuple, str] = (tuple(_activations.keys()), - list(_activations.keys())[0]), - use_dropout: Tuple[Tuple, bool] = ((True, False), False), - dropout: Tuple[Tuple, float] = ((0, 0.5), 0.2) - ) -> ConfigurationSpace: + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict] = None, + num_layers: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='num_layers', + value_range=(4, 64), + default_value=16, + ), + num_blocks: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='num_blocks', + value_range=(3, 4), + default_value=3, + ), + growth_rate: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='growth_rate', + value_range=(12, 40), + default_value=20, + ), + activation: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='activation', + value_range=tuple(_activations.keys()), + default_value=list(_activations.keys())[0], + ), + use_dropout: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='use_dropout', + value_range=(True, False), + default_value=False, + ), + dropout: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter='dropout', + value_range=(0, 0.5), + default_value=0.2, + ), + ) -> ConfigurationSpace: cs = CS.ConfigurationSpace() - min_growth_rate, max_growth_rate = growth_rate[0] - growth_rate_hp = UniformIntegerHyperparameter('growth_rate', - lower=min_growth_rate, - upper=max_growth_rate, - default_value=growth_rate[1]) - cs.add_hyperparameter(growth_rate_hp) - - min_num_blocks, max_num_blocks = num_blocks[0] - blocks_hp = UniformIntegerHyperparameter('blocks', - lower=min_num_blocks, - upper=max_num_blocks, - default_value=num_blocks[1]) + add_hyperparameter(cs, num_layers, UniformIntegerHyperparameter) + add_hyperparameter(cs, growth_rate, UniformIntegerHyperparameter) + + min_num_blocks, max_num_blocks = num_blocks.value_range + blocks_hp = get_hyperparameter(num_blocks, UniformIntegerHyperparameter) cs.add_hyperparameter(blocks_hp) - activation_hp = CategoricalHyperparameter('activation', - choices=activation[0], - default_value=activation[1]) - cs.add_hyperparameter(activation_hp) + add_hyperparameter(cs, activation, CategoricalHyperparameter) - use_dropout = CategoricalHyperparameter('use_dropout', - choices=use_dropout[0], - default_value=use_dropout[1]) + use_dropout = get_hyperparameter(use_dropout, CategoricalHyperparameter) - min_dropout, max_dropout = dropout[0] - dropout = UniformFloatHyperparameter('dropout', - lower=min_dropout, - upper=max_dropout, - default_value=dropout[1]) + dropout = get_hyperparameter(dropout, UniformFloatHyperparameter) cs.add_hyperparameters([use_dropout, dropout]) cs.add_condition(CS.EqualsCondition(dropout, use_dropout, True)) - for i in range(1, max_num_blocks + 1): - min_num_layers, max_num_layers = num_layers[0] - layer_hp = UniformIntegerHyperparameter('layer_in_block_%d' % i, - lower=min_num_layers, - upper=max_num_layers, - default_value=num_layers[1]) - cs.add_hyperparameter(layer_hp) + for i in range(1, int(max_num_blocks) + 1): - if i > min_num_blocks: + layer_search_space = HyperparameterSearchSpace(hyperparameter='layer_in_block_%d' % i, + value_range=num_layers.value_range, + default_value=num_layers.default_value, + log=num_layers.log) + layer_hp = get_hyperparameter(layer_search_space, UniformIntegerHyperparameter) + + cs.add_hyperparameter(layer_hp) + if i > int(min_num_blocks): cs.add_condition(CS.GreaterThanCondition(layer_hp, blocks_hp, i - 1)) return cs diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/InceptionTimeBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/InceptionTimeBackbone.py index 4bf5c8842..e9829b300 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/InceptionTimeBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/InceptionTimeBackbone.py @@ -9,6 +9,7 @@ from torch import nn from autoPyTorch.pipeline.components.setup.network_backbone.base_network_backbone import NetworkBackboneComponent +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter # Code inspired by https://github.com/hfawaz/InceptionTime @@ -142,39 +143,30 @@ def get_properties(dataset_properties: Optional[Dict[str, str]] = None) -> Dict[ } @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, - num_blocks: Tuple[Tuple, int] = ((1, 10), 5), - num_filters: Tuple[Tuple, int] = ((4, 64), 32), - kernel_size: Tuple[Tuple, int] = ((4, 64), 32), - bottleneck_size: Tuple[Tuple, int] = ((16, 64), 32) - ) -> ConfigurationSpace: + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict] = None, + num_blocks: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="num_blocks", + value_range=(1, 10), + default_value=5, + ), + num_filters: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="num_filters", + value_range=(4, 64), + default_value=32, + ), + kernel_size: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="kernel_size", + value_range=(4, 64), + default_value=32, + ), + bottleneck_size: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="bottleneck_size", + value_range=(16, 64), + default_value=32, + ), + ) -> ConfigurationSpace: cs = ConfigurationSpace() - min_num_blocks, max_num_blocks = num_blocks[0] - num_blocks_hp = UniformIntegerHyperparameter("num_blocks", - lower=min_num_blocks, - upper=max_num_blocks, - default_value=num_blocks[1]) - cs.add_hyperparameter(num_blocks_hp) - - min_num_filters, max_num_filters = num_filters[0] - num_filters_hp = UniformIntegerHyperparameter("num_filters", - lower=min_num_filters, - upper=max_num_filters, - default_value=num_filters[1]) - cs.add_hyperparameter(num_filters_hp) - - min_bottleneck_size, max_bottleneck_size = bottleneck_size[0] - bottleneck_size_hp = UniformIntegerHyperparameter("bottleneck_size", - lower=min_bottleneck_size, - upper=max_bottleneck_size, - default_value=bottleneck_size[1]) - cs.add_hyperparameter(bottleneck_size_hp) - - min_kernel_size, max_kernel_size = kernel_size[0] - kernel_size_hp = UniformIntegerHyperparameter("kernel_size", - lower=min_kernel_size, - upper=max_kernel_size, - default_value=kernel_size[1]) - cs.add_hyperparameter(kernel_size_hp) + add_hyperparameter(cs, num_blocks, UniformIntegerHyperparameter) + add_hyperparameter(cs, num_filters, UniformIntegerHyperparameter) + add_hyperparameter(cs, kernel_size, UniformIntegerHyperparameter) + add_hyperparameter(cs, bottleneck_size, UniformIntegerHyperparameter) + return cs diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/MLPBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/MLPBackbone.py index 230ddfe96..930b5acfe 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/MLPBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/MLPBackbone.py @@ -12,6 +12,7 @@ from autoPyTorch.pipeline.components.setup.network_backbone.base_network_backbone import NetworkBackboneComponent from autoPyTorch.pipeline.components.setup.network_backbone.utils import _activations +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter class MLPBackbone(NetworkBackboneComponent): @@ -68,44 +69,53 @@ def get_properties(dataset_properties: Optional[Dict[str, Any]] = None) -> Dict[ } @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, - num_groups: Tuple[Tuple, int] = ((1, 15), 5), - activation: Tuple[Tuple, str] = (tuple(_activations.keys()), - list(_activations.keys())[0]), - use_dropout: Tuple[Tuple, bool] = ((True, False), False), - num_units: Tuple[Tuple, int] = ((10, 1024), 200), - dropout: Tuple[Tuple, float] = ((0, 0.8), 0.5) - ) -> ConfigurationSpace: + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict] = None, + num_groups: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="num_groups", + value_range=(1, 15), + default_value=5, + ), + activation: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="activation", + value_range=tuple(_activations.keys()), + default_value=list(_activations.keys())[0], + ), + use_dropout: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_dropout", + value_range=(True, False), + default_value=False, + ), + num_units: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="num_units", + value_range=(10, 1024), + default_value=200, + ), + dropout: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="dropout", + value_range=(0, 0.8), + default_value=0.5, + ), + ) -> ConfigurationSpace: cs = ConfigurationSpace() # The number of hidden layers the network will have. # Layer blocks are meant to have the same architecture, differing only # by the number of units - min_mlp_layers, max_mlp_layers = num_groups[0] - num_groups = UniformIntegerHyperparameter( - "num_groups", min_mlp_layers, max_mlp_layers, default_value=num_groups[1]) - - activation = CategoricalHyperparameter( - "activation", choices=activation[0], - default_value=activation[1] - ) - cs.add_hyperparameters([num_groups, activation]) + min_mlp_layers, max_mlp_layers = num_groups.value_range + num_groups = get_hyperparameter(num_groups, UniformIntegerHyperparameter) + add_hyperparameter(cs, activation, CategoricalHyperparameter) # We can have dropout in the network for # better generalization - use_dropout = CategoricalHyperparameter( - "use_dropout", choices=use_dropout[0], default_value=use_dropout[1]) - cs.add_hyperparameters([use_dropout]) - - for i in range(1, max_mlp_layers + 1): - n_units_hp = UniformIntegerHyperparameter("num_units_%d" % i, - lower=num_units[0][0], - upper=num_units[0][1], - default_value=num_units[1]) + use_dropout = get_hyperparameter(use_dropout, CategoricalHyperparameter) + cs.add_hyperparameters([num_groups, use_dropout]) + + for i in range(1, int(max_mlp_layers) + 1): + n_units_search_space = HyperparameterSearchSpace(hyperparameter='num_units_%d' % i, + value_range=num_units.value_range, + default_value=num_units.default_value, + log=num_units.log) + n_units_hp = get_hyperparameter(n_units_search_space, UniformIntegerHyperparameter) cs.add_hyperparameter(n_units_hp) - if i > min_mlp_layers: + if i > int(min_mlp_layers): # The units of layer i should only exist # if there are at least i layers cs.add_condition( @@ -113,17 +123,16 @@ def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, n_units_hp, num_groups, i - 1 ) ) - - dropout_hp = UniformFloatHyperparameter( - "dropout_%d" % i, - lower=dropout[0][0], - upper=dropout[0][1], - default_value=dropout[1] - ) + dropout_search_space = HyperparameterSearchSpace(hyperparameter='dropout_%d' % i, + value_range=dropout.value_range, + default_value=dropout.default_value, + log=dropout.log) + dropout_hp = get_hyperparameter(dropout_search_space, UniformFloatHyperparameter) cs.add_hyperparameter(dropout_hp) + dropout_condition_1 = CS.EqualsCondition(dropout_hp, use_dropout, True) - if i > min_mlp_layers: + if i > int(min_mlp_layers): dropout_condition_2 = CS.GreaterThanCondition(dropout_hp, num_groups, i - 1) cs.add_condition(CS.AndConjunction(dropout_condition_1, dropout_condition_2)) else: diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py index 4433f540c..6391baa6a 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py @@ -19,6 +19,7 @@ shake_get_alpha_beta, shake_shake ) +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter class ResNetBackbone(NetworkBackboneComponent): @@ -92,86 +93,100 @@ def get_properties(dataset_properties: Optional[Dict[str, Any]] = None) -> Dict[ } @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, - num_groups: Tuple[Tuple, int] = ((1, 15), 5), - use_dropout: Tuple[Tuple, bool] = ((True, False), False), - num_units: Tuple[Tuple, int] = ((10, 1024), 200), - activation: Tuple[Tuple, str] = (tuple(_activations.keys()), - list(_activations.keys())[0]), - blocks_per_group: Tuple[Tuple, int] = ((1, 4), 2), - dropout: Tuple[Tuple, float] = ((0, 0.8), 0.5), - use_shake_shake: Tuple[Tuple, bool] = ((True, False), True), - use_shake_drop: Tuple[Tuple, bool] = ((True, False), True), - max_shake_drop_probability: Tuple[Tuple, float] = ((0, 1), 0.5) - ) -> ConfigurationSpace: + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict] = None, + num_groups: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="num_groups", + value_range=(1, 15), + default_value=5, + ), + use_dropout: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_dropout", + value_range=(True, False), + default_value=False, + ), + num_units: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="num_units", + value_range=(10, 1024), + default_value=200, + ), + activation: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="activation", + value_range=tuple(_activations.keys()), + default_value=list(_activations.keys())[0], + ), + blocks_per_group: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="blocks_per_group", + value_range=(1, 4), + default_value=2, + ), + dropout: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="dropout", + value_range=(0, 0.8), + default_value=0.5, + ), + use_shake_shake: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_shake_shake", + value_range=(True, False), + default_value=True, + ), + use_shake_drop: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_shake_drop", + value_range=(True, False), + default_value=True, + ), + max_shake_drop_probability: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="max_shake_drop_probability", + value_range=(0, 1), + default_value=0.5), + ) -> ConfigurationSpace: cs = ConfigurationSpace() # The number of groups that will compose the resnet. That is, # a group can have N Resblock. The M number of this N resblock # repetitions is num_groups - min_num_gropus, max_num_groups = num_groups[0] - num_groups = UniformIntegerHyperparameter( - "num_groups", lower=min_num_gropus, upper=max_num_groups, default_value=num_groups[1]) + min_num_gropus, max_num_groups = num_groups.value_range + num_groups = get_hyperparameter(num_groups, UniformIntegerHyperparameter) - activation = CategoricalHyperparameter( - "activation", choices=activation[0], - default_value=activation[1] - ) - cs.add_hyperparameters([num_groups, activation]) + add_hyperparameter(cs, activation, CategoricalHyperparameter) + cs.add_hyperparameters([num_groups]) # We can have dropout in the network for # better generalization - use_dropout = CategoricalHyperparameter("use_dropout", choices=use_dropout[0], default_value=use_dropout[1]) + use_dropout = get_hyperparameter(use_dropout, CategoricalHyperparameter) cs.add_hyperparameters([use_dropout]) - use_shake_shake = CategoricalHyperparameter("use_shake_shake", choices=use_shake_shake[0], - default_value=use_shake_shake[1]) - use_shake_drop = CategoricalHyperparameter("use_shake_drop", choices=use_shake_drop[0], - default_value=use_shake_drop[1]) - shake_drop_prob = UniformFloatHyperparameter( - "max_shake_drop_probability", - lower=max_shake_drop_probability[0][0], - upper=max_shake_drop_probability[0][1], - default_value=max_shake_drop_probability[1]) + use_shake_shake = get_hyperparameter(use_shake_shake, CategoricalHyperparameter) + use_shake_drop = get_hyperparameter(use_shake_drop, CategoricalHyperparameter) + shake_drop_prob = get_hyperparameter(max_shake_drop_probability, UniformFloatHyperparameter) cs.add_hyperparameters([use_shake_shake, use_shake_drop, shake_drop_prob]) cs.add_condition(CS.EqualsCondition(shake_drop_prob, use_shake_drop, True)) # It is the upper bound of the nr of groups, # since the configuration will actually be sampled. - (min_blocks_per_group, max_blocks_per_group), default_blocks_per_group = blocks_per_group[:2] - for i in range(0, max_num_groups + 1): - - n_units = UniformIntegerHyperparameter( - "num_units_%d" % i, - lower=num_units[0][0], - upper=num_units[0][1], - default_value=num_units[1] - ) - blocks_per_group = UniformIntegerHyperparameter( - "blocks_per_group_%d" % i, - lower=min_blocks_per_group, - upper=max_blocks_per_group, - default_value=default_blocks_per_group) + for i in range(0, int(max_num_groups) + 1): + + n_units_search_space = HyperparameterSearchSpace(hyperparameter='num_units_%d' % i, + value_range=num_units.value_range, + default_value=num_units.default_value, + log=num_units.log) + n_units_hp = get_hyperparameter(n_units_search_space, UniformIntegerHyperparameter) - cs.add_hyperparameters([n_units, blocks_per_group]) + blocks_per_group_search_space = HyperparameterSearchSpace(hyperparameter='blocks_per_group_%d' % i, + value_range=blocks_per_group.value_range, + default_value=blocks_per_group.default_value, + log=blocks_per_group.log) + blocks_per_group_hp = get_hyperparameter(blocks_per_group_search_space, UniformIntegerHyperparameter) + cs.add_hyperparameters([n_units_hp, blocks_per_group_hp]) if i > 1: - cs.add_condition(CS.GreaterThanCondition(n_units, num_groups, i - 1)) - cs.add_condition(CS.GreaterThanCondition(blocks_per_group, num_groups, i - 1)) - - this_dropout = UniformFloatHyperparameter( - "dropout_%d" % i, - lower=dropout[0][0], - upper=dropout[0][1], - default_value=dropout[1] - ) - cs.add_hyperparameters([this_dropout]) + cs.add_condition(CS.GreaterThanCondition(n_units_hp, num_groups, i - 1)) + cs.add_condition(CS.GreaterThanCondition(blocks_per_group_hp, num_groups, i - 1)) + + dropout_search_space = HyperparameterSearchSpace(hyperparameter='dropout_%d' % i, + value_range=dropout.value_range, + default_value=dropout.default_value, + log=dropout.log) + dropout_hp = get_hyperparameter(dropout_search_space, UniformFloatHyperparameter) + cs.add_hyperparameter(dropout_hp) - dropout_condition_1 = CS.EqualsCondition(this_dropout, use_dropout, True) + dropout_condition_1 = CS.EqualsCondition(dropout_hp, use_dropout, True) if i > 1: - dropout_condition_2 = CS.GreaterThanCondition(this_dropout, num_groups, i - 1) + dropout_condition_2 = CS.GreaterThanCondition(dropout_hp, num_groups, i - 1) cs.add_condition(CS.AndConjunction(dropout_condition_1, dropout_condition_2)) else: @@ -185,14 +200,14 @@ class ResBlock(nn.Module): """ def __init__( - self, - config: Dict[str, Any], - in_features: int, - out_features: int, - blocks_per_group: int, - block_index: int, - dropout: bool, - activation: nn.Module + self, + config: Dict[str, Any], + in_features: int, + out_features: int, + blocks_per_group: int, + block_index: int, + dropout: bool, + activation: nn.Module ): super(ResBlock, self).__init__() self.config = config diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/ShapedMLPBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/ShapedMLPBackbone.py index 3e8be6b70..c92b6696d 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/ShapedMLPBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/ShapedMLPBackbone.py @@ -15,6 +15,7 @@ _activations, get_shaped_neuron_counts, ) +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter class ShapedMLPBackbone(NetworkBackboneComponent): @@ -75,57 +76,58 @@ def get_properties(dataset_properties: Optional[Dict[str, Any]] = None) -> Dict[ } @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, - num_groups: Tuple[Tuple, int] = ((1, 15), 5), - max_dropout: Tuple[Tuple, float] = ((0, 1), 0.5), - use_dropout: Tuple[Tuple, bool] = ((True, False), False), - max_units: Tuple[Tuple, int] = ((10, 1024), 200), - output_dim: Tuple[Tuple, int] = ((10, 1024), 200), - mlp_shape: Tuple[Tuple, str] = (('funnel', 'long_funnel', - 'diamond', 'hexagon', - 'brick', 'triangle', 'stairs'), 'funnel'), - activation: Tuple[Tuple, str] = ( - tuple(_activations.keys()), list(_activations.keys())[0]) - ) -> ConfigurationSpace: + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict] = None, + num_groups: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="num_groups", + value_range=(1, 15), + default_value=5, + ), + max_dropout: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="max_dropout", + value_range=(0, 1), + default_value=0.5, + ), + use_dropout: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_dropout", + value_range=(True, False), + default_value=False, + ), + max_units: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="max_units", + value_range=(10, 1024), + default_value=200, + ), + output_dim: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="output_dim", + value_range=(10, 1024), + default_value=200, + ), + mlp_shape: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="mlp_shape", + value_range=('funnel', 'long_funnel', + 'diamond', 'hexagon', + 'brick', 'triangle', + 'stairs'), + default_value='funnel', + ), + activation: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="activation", + value_range=tuple(_activations.keys()), + default_value=list(_activations.keys())[0], + ), + + ) -> ConfigurationSpace: cs = ConfigurationSpace() # The number of groups that will compose the resnet. That is, # a group can have N Resblock. The M number of this N resblock # repetitions is num_groups - num_groups = UniformIntegerHyperparameter( - "num_groups", lower=num_groups[0][0], upper=num_groups[0][1], default_value=num_groups[1]) - - mlp_shape = CategoricalHyperparameter('mlp_shape', choices=mlp_shape[0], - default_value=mlp_shape[1]) - - activation = CategoricalHyperparameter( - "activation", choices=activation[0], - default_value=activation[1] - ) - (min_num_units, max_num_units), default_units = max_units[:2] - max_units = UniformIntegerHyperparameter( - "max_units", - lower=min_num_units, - upper=max_num_units, - default_value=default_units, - ) - - output_dim = UniformIntegerHyperparameter( - "output_dim", - lower=output_dim[0][0], - upper=output_dim[0][1], - default_value=output_dim[1] - ) - - cs.add_hyperparameters([num_groups, activation, mlp_shape, max_units, output_dim]) + add_hyperparameter(cs, num_groups, UniformIntegerHyperparameter) + add_hyperparameter(cs, mlp_shape, CategoricalHyperparameter) + add_hyperparameter(cs, activation, CategoricalHyperparameter) + add_hyperparameter(cs, max_units, UniformIntegerHyperparameter) + add_hyperparameter(cs, output_dim, UniformIntegerHyperparameter) # We can have dropout in the network for # better generalization - use_dropout = CategoricalHyperparameter( - "use_dropout", choices=use_dropout[0], default_value=use_dropout[1]) - max_dropout = UniformFloatHyperparameter("max_dropout", lower=max_dropout[0][0], upper=max_dropout[0][1], - default_value=max_dropout[1]) + use_dropout = get_hyperparameter(use_dropout, CategoricalHyperparameter) + max_dropout = get_hyperparameter(max_dropout, UniformFloatHyperparameter) + cs.add_hyperparameters([use_dropout, max_dropout]) cs.add_condition(CS.EqualsCondition(max_dropout, use_dropout, True)) diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py index fbfae28ad..c8ffa1b4e 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py @@ -15,6 +15,7 @@ _activations, get_shaped_neuron_counts, ) +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter class ShapedResNetBackbone(ResNetBackbone): @@ -80,81 +81,76 @@ def get_properties(dataset_properties: Optional[Dict[str, Any]] = None) -> Dict[ } @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, # type: ignore[override] - num_groups: Tuple[Tuple, int] = ((1, 15), 5), - use_dropout: Tuple[Tuple, bool] = ((True, False), False), - max_units: Tuple[Tuple, int] = ((10, 1024), 200), - blocks_per_group: Tuple[Tuple, int] = ((1, 4), 2), - max_dropout: Tuple[Tuple, float] = ((0, 0.8), 0.5), - use_shake_shake: Tuple[Tuple, bool] = ((True, False), True), - use_shake_drop: Tuple[Tuple, bool] = ((True, False), True), - max_shake_drop_probability: Tuple[Tuple, float] = ((0, 1), 0.5), - resnet_shape: Tuple[Tuple, str] = (('funnel', 'long_funnel', - 'diamond', 'hexagon', - 'brick', 'triangle', 'stairs'), 'funnel'), - activation: Tuple[Tuple, str] = ( - tuple(_activations.keys()), list(_activations.keys())[0]), - output_dim: Tuple[Tuple, int] = ((10, 1024), 200), - ) -> ConfigurationSpace: + def get_hyperparameter_search_space( # type: ignore[override] + dataset_properties: Optional[Dict] = None, + resnet_shape: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="resnet_shape", + value_range=('funnel', 'long_funnel', + 'diamond', 'hexagon', + 'brick', 'triangle', + 'stairs'), + default_value='funnel', + ), + output_dim: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="output_dim", + value_range=(10, 1024), + default_value=200, + ), + num_groups: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="num_groups", + value_range=(1, 15), + default_value=5, + ), + use_dropout: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_dropout", + value_range=(True, False), + default_value=False, + ), + max_units: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="max_units", + value_range=(10, 1024), + default_value=200), + activation: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="activation", + value_range=tuple(_activations.keys()), + default_value=list(_activations.keys())[0]), + blocks_per_group: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="blocks_per_group", + value_range=(1, 4), + default_value=2), + max_dropout: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="max_dropout", + value_range=(0, 0.8), + default_value=0.5), + use_shake_shake: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_shake_shake", + value_range=(True, False), + default_value=True), + use_shake_drop: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_shake_drop", + value_range=(True, False), + default_value=True), + max_shake_drop_probability: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="max_shake_drop_probability", + value_range=(0, 1), + default_value=0.5), + ) -> ConfigurationSpace: + cs = ConfigurationSpace() # Support for different shapes - resnet_shape = CategoricalHyperparameter( - 'resnet_shape', - choices=resnet_shape[0], - default_value=resnet_shape[1] - ) - cs.add_hyperparameter(resnet_shape) + add_hyperparameter(cs, resnet_shape, CategoricalHyperparameter) # The number of groups that will compose the resnet. That is, # a group can have N Resblock. The M number of this N resblock # repetitions is num_groups - num_groups = UniformIntegerHyperparameter( - "num_groups", lower=num_groups[0][0], upper=num_groups[0][1], default_value=num_groups[1]) + add_hyperparameter(cs, num_groups, UniformIntegerHyperparameter) + add_hyperparameter(cs, blocks_per_group, UniformIntegerHyperparameter) - blocks_per_group = UniformIntegerHyperparameter( - "blocks_per_group", lower=blocks_per_group[0][0], - upper=blocks_per_group[0][1], - default_value=blocks_per_group[1]) + add_hyperparameter(cs, activation, CategoricalHyperparameter) + add_hyperparameter(cs, output_dim, UniformIntegerHyperparameter) - activation = CategoricalHyperparameter( - "activation", choices=activation[0], - default_value=activation[1] - ) - (min_num_units, max_num_units), default_units = max_units[:2] - output_dim = UniformIntegerHyperparameter( - "output_dim", - lower=output_dim[0][0], - upper=output_dim[0][1], - default_value=output_dim[1] - ) - - cs.add_hyperparameters([num_groups, blocks_per_group, activation, output_dim]) - - use_shake_shake = CategoricalHyperparameter("use_shake_shake", choices=use_shake_shake[0], - default_value=use_shake_shake[1]) - use_shake_drop = CategoricalHyperparameter("use_shake_drop", choices=use_shake_drop[0], - default_value=use_shake_drop[1]) - shake_drop_prob = UniformFloatHyperparameter( - "max_shake_drop_probability", - lower=max_shake_drop_probability[0][0], - upper=max_shake_drop_probability[0][1], - default_value=max_shake_drop_probability[1]) + use_shake_shake = get_hyperparameter(use_shake_shake, CategoricalHyperparameter) + use_shake_drop = get_hyperparameter(use_shake_drop, CategoricalHyperparameter) + shake_drop_prob = get_hyperparameter(max_shake_drop_probability, UniformFloatHyperparameter) cs.add_hyperparameters([use_shake_shake, use_shake_drop, shake_drop_prob]) cs.add_condition(CS.EqualsCondition(shake_drop_prob, use_shake_drop, True)) - max_units = UniformIntegerHyperparameter( - "max_units", - lower=min_num_units, - upper=max_num_units, - default_value=default_units - ) - cs.add_hyperparameters([max_units]) + add_hyperparameter(cs, max_units, UniformIntegerHyperparameter) + + use_dropout = get_hyperparameter(use_dropout, CategoricalHyperparameter) + max_dropout = get_hyperparameter(max_dropout, UniformFloatHyperparameter) - use_dropout = CategoricalHyperparameter( - "use_dropout", choices=use_dropout[0], default_value=use_dropout[1]) - max_dropout = UniformFloatHyperparameter("max_dropout", lower=max_dropout[0][0], upper=max_dropout[0][1], - default_value=max_dropout[1]) cs.add_hyperparameters([use_dropout]) cs.add_hyperparameters([max_dropout]) cs.add_condition(CS.EqualsCondition(max_dropout, use_dropout, True)) diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/TCNBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/TCNBackbone.py index c9768153f..b7129c226 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/TCNBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/TCNBackbone.py @@ -13,6 +13,7 @@ from torch.nn.utils import weight_norm from autoPyTorch.pipeline.components.setup.network_backbone.base_network_backbone import NetworkBackboneComponent +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter # _Chomp1d, _TemporalBlock and _TemporalConvNet copied from @@ -122,50 +123,47 @@ def get_properties(dataset_properties: Optional[Dict[str, str]] = None) -> Dict[ } @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, - num_blocks: Tuple[Tuple, int] = ((1, 10), 5), - num_filters: Tuple[Tuple, int] = ((4, 64), 32), - kernel_size: Tuple[Tuple, int] = ((4, 64), 32), - use_dropout: Tuple[Tuple, bool] = ((True, False), False), - dropout: Tuple[Tuple, float] = ((0.0, 0.5), 0.1) - ) -> ConfigurationSpace: + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict] = None, + num_blocks: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="num_blocks", + value_range=(1, 10), + default_value=5), + num_filters: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="num_filters", + value_range=(4, 64), + default_value=32), + kernel_size: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="kernel_size", + value_range=(4, 64), + default_value=32), + use_dropout: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_dropout", + value_range=(True, False), + default_value=False), + dropout: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="dropout", + value_range=(0, 0.5), + default_value=0.1), + ) -> ConfigurationSpace: cs = ConfigurationSpace() - min_num_blocks, max_num_blocks = num_blocks[0] - num_blocks_hp = UniformIntegerHyperparameter("num_blocks", - lower=min_num_blocks, - upper=max_num_blocks, - default_value=num_blocks[1]) + min_num_blocks, max_num_blocks = num_blocks.value_range + num_blocks_hp = get_hyperparameter(num_blocks, UniformIntegerHyperparameter) cs.add_hyperparameter(num_blocks_hp) - min_kernel_size, max_kernel_size = kernel_size[0] - kernel_size_hp = UniformIntegerHyperparameter("kernel_size", - lower=min_kernel_size, - upper=max_kernel_size, - default_value=kernel_size[1]) - cs.add_hyperparameter(kernel_size_hp) + add_hyperparameter(cs, kernel_size, UniformIntegerHyperparameter) - use_dropout_hp = CategoricalHyperparameter("use_dropout", - choices=use_dropout[0], - default_value=use_dropout[1]) + use_dropout_hp = get_hyperparameter(use_dropout, CategoricalHyperparameter) cs.add_hyperparameter(use_dropout_hp) - min_dropout, max_dropout = dropout[0] - dropout_hp = UniformFloatHyperparameter("dropout", - lower=min_dropout, - upper=max_dropout, - default_value=dropout[1]) + dropout_hp = get_hyperparameter(dropout, UniformFloatHyperparameter) cs.add_hyperparameter(dropout_hp) cs.add_condition(CS.EqualsCondition(dropout_hp, use_dropout_hp, True)) - for i in range(0, max_num_blocks): - min_num_filters, max_num_filters = num_filters[0] - num_filters_hp = UniformIntegerHyperparameter(f"num_filters_{i}", - lower=min_num_filters, - upper=max_num_filters, - default_value=num_filters[1]) + for i in range(0, int(max_num_blocks)): + num_filter_search_space = HyperparameterSearchSpace(f"num_filters_{i}", + value_range=num_filters.value_range, + default_value=num_filters.default_value, + log=num_filters.log) + num_filters_hp = get_hyperparameter(num_filter_search_space, UniformIntegerHyperparameter) cs.add_hyperparameter(num_filters_hp) - if i >= min_num_blocks: + if i >= int(min_num_blocks): cs.add_condition(CS.GreaterThanCondition( num_filters_hp, num_blocks_hp, i)) diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/base_network_backbone_choice.py b/autoPyTorch/pipeline/components/setup/network_backbone/base_network_backbone_choice.py index 71d8a63d3..13793c393 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/base_network_backbone_choice.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/base_network_backbone_choice.py @@ -158,14 +158,25 @@ def get_hyperparameter_search_space( if default_ in available_backbones: default = default_ break - - backbone = CSH.CategoricalHyperparameter( - '__choice__', - list(available_backbones.keys()), - default_value=default - ) + updates = self._get_search_space_updates() + if '__choice__' in updates.keys(): + choice_hyperparameter = updates['__choice__'] + if not set(choice_hyperparameter.value_range).issubset(available_backbones): + raise ValueError("Expected given update for {} to have " + "choices in {} got {}".format(self.__class__.__name__, + available_backbones, + choice_hyperparameter.value_range)) + backbone = CSH.CategoricalHyperparameter('__choice__', + choice_hyperparameter.value_range, + default_value=choice_hyperparameter.default_value) + else: + backbone = CSH.CategoricalHyperparameter( + '__choice__', + list(available_backbones.keys()), + default_value=default + ) cs.add_hyperparameter(backbone) - for name in available_backbones: + for name in backbone.choices: updates = self._get_search_space_updates(prefix=name) config_space = available_backbones[name].get_hyperparameter_search_space(dataset_properties, # type: ignore **updates) diff --git a/autoPyTorch/pipeline/components/setup/network_embedding/LearnedEntityEmbedding.py b/autoPyTorch/pipeline/components/setup/network_embedding/LearnedEntityEmbedding.py index 3910afc37..648c63e0c 100644 --- a/autoPyTorch/pipeline/components/setup/network_embedding/LearnedEntityEmbedding.py +++ b/autoPyTorch/pipeline/components/setup/network_embedding/LearnedEntityEmbedding.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, Optional, Union from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( @@ -12,6 +12,7 @@ from torch import nn from autoPyTorch.pipeline.components.setup.network_embedding.base_network_embedding import NetworkEmbeddingComponent +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter class _LearnedEntityEmbedding(nn.Module): @@ -100,25 +101,24 @@ def build_embedding(self, num_input_features: np.ndarray, num_numerical_features @staticmethod def get_hyperparameter_search_space( dataset_properties: Optional[Dict[str, str]] = None, - min_unique_values_for_embedding: Tuple[Tuple, int, bool] = ((3, 7), 5, True), - dimension_reduction: Tuple[Tuple, float] = ((0, 1), 0.5), + min_unique_values_for_embedding: HyperparameterSearchSpace = HyperparameterSearchSpace( + hyperparameter="min_unique_values_for_embedding", + value_range=(3, 7), + default_value=5, + log=True), + dimension_reduction: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="dimension_reduction", + value_range=(0, 1), + default_value=0.5), ) -> ConfigurationSpace: cs = ConfigurationSpace() - min_hp = UniformIntegerHyperparameter("min_unique_values_for_embedding", - lower=min_unique_values_for_embedding[0][0], - upper=min_unique_values_for_embedding[0][1], - default_value=min_unique_values_for_embedding[1], - log=min_unique_values_for_embedding[2] - ) - cs.add_hyperparameter(min_hp) + add_hyperparameter(cs, min_unique_values_for_embedding, UniformIntegerHyperparameter) if dataset_properties is not None: for i in range(len(dataset_properties['categorical_columns'])): - ee_dimensions_hp = UniformFloatHyperparameter("dimension_reduction_" + str(i), - lower=dimension_reduction[0][0], - upper=dimension_reduction[0][1], - default_value=dimension_reduction[1] - ) - cs.add_hyperparameter(ee_dimensions_hp) + ee_dimensions_search_space = HyperparameterSearchSpace(hyperparameter="dimension_reduction_" + str(i), + value_range=dimension_reduction.value_range, + default_value=dimension_reduction.default_value, + log=dimension_reduction.log) + add_hyperparameter(cs, ee_dimensions_search_space, UniformFloatHyperparameter) return cs @staticmethod diff --git a/autoPyTorch/pipeline/components/setup/network_embedding/base_network_embedding_choice.py b/autoPyTorch/pipeline/components/setup/network_embedding/base_network_embedding_choice.py index c08b156ce..14a5c93d9 100644 --- a/autoPyTorch/pipeline/components/setup/network_embedding/base_network_embedding_choice.py +++ b/autoPyTorch/pipeline/components/setup/network_embedding/base_network_embedding_choice.py @@ -146,26 +146,42 @@ def get_hyperparameter_search_space( if default is None: defaults = [ + 'NoEmbedding', 'LearnedEntityEmbedding', - 'NoEmbedding' ] for default_ in defaults: if default_ in available_embedding: default = default_ break - - if len(dataset_properties['categorical_columns']) == 0: - default = 'NoEmbedding' - if include is not None and default not in include: - raise ValueError("Provided {} in include, however, the dataset " - "is incompatible with it".format(include)) + updates = self._get_search_space_updates() + if '__choice__' in updates.keys(): + choice_hyperparameter = updates['__choice__'] + if not set(choice_hyperparameter.value_range).issubset(available_embedding): + raise ValueError("Expected given update for {} to have " + "choices in {} got {}".format(self.__class__.__name__, + available_embedding, + choice_hyperparameter.value_range)) + if len(dataset_properties['categorical_columns']) == 0: + assert len(choice_hyperparameter.value_range) == 1 + if 'NoEmbedding' not in choice_hyperparameter.value_range: + raise ValueError("Provided {} in choices, however, the dataset " + "is incompatible with it".format(choice_hyperparameter.value_range)) embedding = CSH.CategoricalHyperparameter('__choice__', - ['NoEmbedding'], - default_value=default) + choice_hyperparameter.value_range, + default_value=choice_hyperparameter.default_value) else: - embedding = CSH.CategoricalHyperparameter('__choice__', - list(available_embedding.keys()), - default_value=default) + if len(dataset_properties['categorical_columns']) == 0: + default = 'NoEmbedding' + if include is not None and default not in include: + raise ValueError("Provided {} in include, however, the dataset " + "is incompatible with it".format(include)) + embedding = CSH.CategoricalHyperparameter('__choice__', + ['NoEmbedding'], + default_value=default) + else: + embedding = CSH.CategoricalHyperparameter('__choice__', + list(available_embedding.keys()), + default_value=default) cs.add_hyperparameter(embedding) for name in embedding.choices: diff --git a/autoPyTorch/pipeline/components/setup/network_head/base_network_head_choice.py b/autoPyTorch/pipeline/components/setup/network_head/base_network_head_choice.py index f57e60b91..c03e860fc 100644 --- a/autoPyTorch/pipeline/components/setup/network_head/base_network_head_choice.py +++ b/autoPyTorch/pipeline/components/setup/network_head/base_network_head_choice.py @@ -17,7 +17,6 @@ NetworkHeadComponent, ) - directory = os.path.split(__file__)[0] _heads = find_components(__package__, directory, @@ -157,14 +156,24 @@ def get_hyperparameter_search_space( if default_ in available_heads: default = default_ break - - head = CSH.CategoricalHyperparameter( - '__choice__', - list(available_heads.keys()), - default_value=default - ) + updates = self._get_search_space_updates() + if '__choice__' in updates.keys(): + choice_hyperparameter = updates['__choice__'] + if not set(choice_hyperparameter.value_range).issubset(available_heads): + raise ValueError("Expected given update for {} to have " + "choices in {} got {}".format(self.__class__.__name__, + available_heads, + choice_hyperparameter.value_range)) + head = CSH.CategoricalHyperparameter('__choice__', + choice_hyperparameter.value_range, + default_value=choice_hyperparameter.default_value) + else: + head = CSH.CategoricalHyperparameter( + '__choice__', + list(available_heads.keys()), + default_value=default) cs.add_hyperparameter(head) - for name in available_heads: + for name in head.choices: updates = self._get_search_space_updates(prefix=name) config_space = available_heads[name].get_hyperparameter_search_space(dataset_properties, # type: ignore **updates) diff --git a/autoPyTorch/pipeline/components/setup/network_head/fully_connected.py b/autoPyTorch/pipeline/components/setup/network_head/fully_connected.py index f01839234..93f653b37 100644 --- a/autoPyTorch/pipeline/components/setup/network_head/fully_connected.py +++ b/autoPyTorch/pipeline/components/setup/network_head/fully_connected.py @@ -10,6 +10,7 @@ from autoPyTorch.pipeline.components.setup.network_head.base_network_head import NetworkHeadComponent from autoPyTorch.pipeline.components.setup.network_head.utils import _activations +from autoPyTorch.utils.common import HyperparameterSearchSpace, get_hyperparameter class FullyConnectedHead(NetworkHeadComponent): @@ -42,37 +43,37 @@ def get_properties(dataset_properties: Optional[Dict[str, Any]] = None) -> Dict[ } @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict[str, str]] = None, - num_layers: Tuple[Tuple, int] = ((1, 4), 2), - units_layer: Tuple[Tuple, int] = ((64, 512), 128), - activation: Tuple[Tuple, str] = (tuple(_activations.keys()), - list(_activations.keys())[0]) - ) -> ConfigurationSpace: + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict[str, str]] = None, + num_layers: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="num_layers", + value_range=(1, 4), + default_value=2), + units_layer: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="units_layer", + value_range=(64, 512), + default_value=128), + activation: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="activation", + value_range=tuple(_activations.keys()), + default_value=list(_activations.keys())[0]), + ) -> ConfigurationSpace: cs = ConfigurationSpace() - min_num_layers, max_num_layers = num_layers[0] - num_layers_hp = UniformIntegerHyperparameter("num_layers", - lower=min_num_layers, - upper=max_num_layers, - default_value=num_layers[1] - ) + min_num_layers, max_num_layers = num_layers.value_range + num_layers_hp = get_hyperparameter(num_layers, UniformIntegerHyperparameter) - activation_hp = CategoricalHyperparameter( - "activation", choices=activation[0], - default_value=activation[1] - ) + activation_hp = get_hyperparameter(activation, CategoricalHyperparameter) cs.add_hyperparameters([num_layers_hp, activation_hp]) cs.add_condition(CS.GreaterThanCondition(activation_hp, num_layers_hp, 1)) - for i in range(1, max_num_layers): - - num_units_hp = UniformIntegerHyperparameter(f"units_layer_{i}", - lower=units_layer[0][0], - upper=units_layer[0][1], - default_value=units_layer[1]) + for i in range(1, int(max_num_layers)): + num_units_search_space = HyperparameterSearchSpace(hyperparameter=f"units_layer_{i}", + value_range=units_layer.value_range, + default_value=units_layer.default_value, + log=units_layer.log) + num_units_hp = get_hyperparameter(num_units_search_space, UniformIntegerHyperparameter) cs.add_hyperparameter(num_units_hp) - if i >= min_num_layers: + + if i >= int(min_num_layers): cs.add_condition(CS.GreaterThanCondition(num_units_hp, num_layers_hp, i)) return cs diff --git a/autoPyTorch/pipeline/components/setup/network_head/fully_convolutional.py b/autoPyTorch/pipeline/components/setup/network_head/fully_convolutional.py index 21ae3eb71..e291442ea 100644 --- a/autoPyTorch/pipeline/components/setup/network_head/fully_convolutional.py +++ b/autoPyTorch/pipeline/components/setup/network_head/fully_convolutional.py @@ -9,6 +9,7 @@ from autoPyTorch.pipeline.components.setup.network_head.base_network_head import NetworkHeadComponent from autoPyTorch.pipeline.components.setup.network_head.utils import _activations +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter class _FullyConvolutional2DHead(nn.Module): @@ -70,32 +71,41 @@ def get_properties(dataset_properties: Optional[Dict[str, Any]] = None) -> Dict[ } @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict[str, str]] = None, - min_num_layers: int = 1, - max_num_layers: int = 4, - min_num_filters: int = 16, - max_num_filters: int = 256) -> ConfigurationSpace: + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict[str, str]] = None, + num_layers: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="num_layers", + value_range=(1, 4), + default_value=2), + num_filters: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="num_filters", + value_range=(16, 256), + default_value=32), + activation: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="activation", + value_range=tuple(_activations.keys()), + default_value=list(_activations.keys())[0]), + pooling_method: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="pooling_method", + value_range=("average", "max"), + default_value="max"), + ) -> ConfigurationSpace: cs = ConfigurationSpace() - num_layers_hp = UniformIntegerHyperparameter("num_layers", - lower=min_num_layers, - upper=max_num_layers) + min_num_layers, max_num_layers = num_layers.value_range + num_layers_hp = get_hyperparameter(num_layers, UniformIntegerHyperparameter) - pooling_method_hp = CategoricalHyperparameter("pooling_method", - choices=["average", "max"]) + add_hyperparameter(cs, pooling_method, CategoricalHyperparameter) - activation_hp = CategoricalHyperparameter('activation', - choices=list(_activations.keys())) + activation_hp = get_hyperparameter(activation, CategoricalHyperparameter) - cs.add_hyperparameters([num_layers_hp, pooling_method_hp, activation_hp]) + cs.add_hyperparameters([num_layers_hp, activation_hp]) cs.add_condition(CS.GreaterThanCondition(activation_hp, num_layers_hp, 1)) - for i in range(1, max_num_layers): - num_filters_hp = UniformIntegerHyperparameter(f"layer_{i}_filters", - lower=min_num_filters, - upper=max_num_filters) + for i in range(1, int(max_num_layers)): + num_filters_search_space = HyperparameterSearchSpace(f"layer_{i}_filters", + value_range=num_filters.value_range, + default_value=num_filters.default_value, + log=num_filters.log) + num_filters_hp = get_hyperparameter(num_filters_search_space, UniformIntegerHyperparameter) cs.add_hyperparameter(num_filters_hp) - if i >= min_num_layers: + if i >= int(min_num_layers): cs.add_condition(CS.GreaterThanCondition(num_filters_hp, num_layers_hp, i)) return cs diff --git a/autoPyTorch/pipeline/components/setup/network_initializer/base_network_init_choice.py b/autoPyTorch/pipeline/components/setup/network_initializer/base_network_init_choice.py index ea9b4c1d9..cc7dfcfc6 100644 --- a/autoPyTorch/pipeline/components/setup/network_initializer/base_network_init_choice.py +++ b/autoPyTorch/pipeline/components/setup/network_initializer/base_network_init_choice.py @@ -46,10 +46,10 @@ def get_components(self) -> Dict[str, autoPyTorchComponent]: return components def get_available_components( - self, - dataset_properties: Optional[Dict[str, str]] = None, - include: List[str] = None, - exclude: List[str] = None, + self, + dataset_properties: Optional[Dict[str, str]] = None, + include: List[str] = None, + exclude: List[str] = None, ) -> Dict[str, autoPyTorchComponent]: """Filters out components based on user provided include/exclude directives, as well as the dataset properties @@ -102,11 +102,11 @@ def get_available_components( return components_dict def get_hyperparameter_search_space( - self, - dataset_properties: Optional[Dict[str, str]] = None, - default: Optional[str] = None, - include: Optional[List[str]] = None, - exclude: Optional[List[str]] = None, + self, + dataset_properties: Optional[Dict[str, str]] = None, + default: Optional[str] = None, + include: Optional[List[str]] = None, + exclude: Optional[List[str]] = None, ) -> ConfigurationSpace: """Returns the configuration space of the current chosen components @@ -141,14 +141,25 @@ def get_hyperparameter_search_space( if default_ in initializers: default = default_ break - - initializer = CSH.CategoricalHyperparameter( - '__choice__', - list(initializers.keys()), - default_value=default - ) + updates = self._get_search_space_updates() + if '__choice__' in updates.keys(): + choice_hyperparameter = updates['__choice__'] + if not set(choice_hyperparameter.value_range).issubset(initializers): + raise ValueError("Expected given update for {} to have " + "choices in {} got {}".format(self.__class__.__name__, + initializers, + choice_hyperparameter.value_range)) + initializer = CSH.CategoricalHyperparameter('__choice__', + choice_hyperparameter.value_range, + default_value=choice_hyperparameter.default_value) + else: + initializer = CSH.CategoricalHyperparameter( + '__choice__', + list(initializers.keys()), + default_value=default + ) cs.add_hyperparameter(initializer) - for name in initializers: + for name in initializer.choices: updates = self._get_search_space_updates(prefix=name) config_space = initializers[name].get_hyperparameter_search_space(dataset_properties, # type:ignore **updates) diff --git a/autoPyTorch/pipeline/components/setup/network_initializer/base_network_initializer.py b/autoPyTorch/pipeline/components/setup/network_initializer/base_network_initializer.py index 93f478f1d..c35c812e7 100644 --- a/autoPyTorch/pipeline/components/setup/network_initializer/base_network_initializer.py +++ b/autoPyTorch/pipeline/components/setup/network_initializer/base_network_initializer.py @@ -1,5 +1,5 @@ from abc import abstractmethod -from typing import Any, Callable, Dict, Optional, Tuple +from typing import Any, Callable, Dict, Optional from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( @@ -11,7 +11,7 @@ import torch from autoPyTorch.pipeline.components.setup.base_setup import autoPyTorchSetupComponent -from autoPyTorch.utils.common import FitRequirement +from autoPyTorch.utils.common import FitRequirement, HyperparameterSearchSpace, add_hyperparameter class BaseNetworkInitializerComponent(autoPyTorchSetupComponent): @@ -72,16 +72,16 @@ def transform(self, X: Dict[str, Any]) -> Dict[str, Any]: return X @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, - bias_strategy: Tuple[Tuple, str] = (('Zero', 'Normal'), 'Normal') - ) -> ConfigurationSpace: - + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict] = None, + bias_strategy: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="bias_strategy", + value_range=('Zero', 'Normal'), + default_value='Normal') + ) -> ConfigurationSpace: cs = ConfigurationSpace() # The strategy for bias initializations - bias_strategy = CategoricalHyperparameter( - "bias_strategy", choices=bias_strategy[0], default_value=bias_strategy[1]) - cs.add_hyperparameters([bias_strategy]) + add_hyperparameter(cs, bias_strategy, CategoricalHyperparameter) return cs def __str__(self) -> str: diff --git a/autoPyTorch/pipeline/components/setup/optimizer/AdamOptimizer.py b/autoPyTorch/pipeline/components/setup/optimizer/AdamOptimizer.py index aa86e9d0c..b1abbe7e6 100644 --- a/autoPyTorch/pipeline/components/setup/optimizer/AdamOptimizer.py +++ b/autoPyTorch/pipeline/components/setup/optimizer/AdamOptimizer.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple +from typing import Any, Dict, Optional from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( @@ -10,6 +10,7 @@ from torch.optim import Adam from autoPyTorch.pipeline.components.setup.optimizer.base_optimizer import BaseOptimizerComponent +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter class AdamOptimizer(BaseOptimizerComponent): @@ -32,7 +33,6 @@ def __init__( weight_decay: float, random_state: Optional[np.random.RandomState] = None, ): - super().__init__() self.lr = lr self.beta1 = beta1 @@ -73,28 +73,28 @@ def get_properties(dataset_properties: Optional[Dict[str, Any]] = None) -> Dict[ } @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, - lr: Tuple[Tuple, float, bool] = ((1e-5, 1e-1), 1e-2, True), - beta1: Tuple[Tuple, float] = ((0.85, 0.999), 0.9), - beta2: Tuple[Tuple, float] = ((0.9, 0.9999), 0.9), - weight_decay: Tuple[Tuple, float] = ((0.0, 0.1), 0.0) - ) -> ConfigurationSpace: - + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict] = None, + lr: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="lr", + value_range=(1e-5, 1e-1), + default_value=1e-2, + log=True), + beta1: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="beta1", + value_range=(0.85, 0.999), + default_value=0.9), + beta2: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="beta2", + value_range=(0.9, 0.9999), + default_value=0.9), + weight_decay: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="weight_decay", + value_range=(0.0, 0.1), + default_value=0.0), + ) -> ConfigurationSpace: cs = ConfigurationSpace() # The learning rate for the model - lr = UniformFloatHyperparameter('lr', lower=lr[0][0], upper=lr[0][1], - default_value=lr[1], log=lr[2]) - - beta1 = UniformFloatHyperparameter('beta1', lower=beta1[0][0], upper=beta1[0][1], - default_value=beta1[1]) - - beta2 = UniformFloatHyperparameter('beta2', lower=beta2[0][0], upper=beta2[0][1], - default_value=beta2[1]) - - weight_decay = UniformFloatHyperparameter('weight_decay', lower=weight_decay[0][0], upper=weight_decay[0][1], - default_value=weight_decay[1]) - - cs.add_hyperparameters([lr, beta1, beta2, weight_decay]) + add_hyperparameter(cs, lr, UniformFloatHyperparameter) + add_hyperparameter(cs, beta1, UniformFloatHyperparameter) + add_hyperparameter(cs, beta2, UniformFloatHyperparameter) + add_hyperparameter(cs, weight_decay, UniformFloatHyperparameter) return cs diff --git a/autoPyTorch/pipeline/components/setup/optimizer/AdamWOptimizer.py b/autoPyTorch/pipeline/components/setup/optimizer/AdamWOptimizer.py index b525cdef7..658b378c8 100644 --- a/autoPyTorch/pipeline/components/setup/optimizer/AdamWOptimizer.py +++ b/autoPyTorch/pipeline/components/setup/optimizer/AdamWOptimizer.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple +from typing import Any, Dict, Optional from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( @@ -10,6 +10,7 @@ from torch.optim import AdamW from autoPyTorch.pipeline.components.setup.optimizer.base_optimizer import BaseOptimizerComponent +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter class AdamWOptimizer(BaseOptimizerComponent): @@ -32,7 +33,6 @@ def __init__( weight_decay: float, random_state: Optional[np.random.RandomState] = None, ): - super().__init__() self.lr = lr self.beta1 = beta1 @@ -73,28 +73,28 @@ def get_properties(dataset_properties: Optional[Dict[str, Any]] = None) -> Dict[ } @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, - lr: Tuple[Tuple, float, bool] = ((1e-5, 1e-1), 1e-2, True), - beta1: Tuple[Tuple, float] = ((0.85, 0.999), 0.9), - beta2: Tuple[Tuple, float] = ((0.9, 0.9999), 0.9), - weight_decay: Tuple[Tuple, float] = ((0.0, 0.1), 0.0) - ) -> ConfigurationSpace: - + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict] = None, + lr: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="lr", + value_range=(1e-5, 1e-1), + default_value=1e-2, + log=True), + beta1: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="beta1", + value_range=(0.85, 0.999), + default_value=0.9), + beta2: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="beta2", + value_range=(0.9, 0.9999), + default_value=0.9), + weight_decay: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="weight_decay", + value_range=(0.0, 0.1), + default_value=0.0), + ) -> ConfigurationSpace: cs = ConfigurationSpace() # The learning rate for the model - lr = UniformFloatHyperparameter('lr', lower=lr[0][0], upper=lr[0][1], - default_value=lr[1], log=lr[2]) - - beta1 = UniformFloatHyperparameter('beta1', lower=beta1[0][0], upper=beta1[0][1], - default_value=beta1[1]) - - beta2 = UniformFloatHyperparameter('beta2', lower=beta2[0][0], upper=beta2[0][1], - default_value=beta2[1]) - - weight_decay = UniformFloatHyperparameter('weight_decay', lower=weight_decay[0][0], upper=weight_decay[0][1], - default_value=weight_decay[1]) - - cs.add_hyperparameters([lr, beta1, beta2, weight_decay]) + add_hyperparameter(cs, lr, UniformFloatHyperparameter) + add_hyperparameter(cs, beta1, UniformFloatHyperparameter) + add_hyperparameter(cs, beta2, UniformFloatHyperparameter) + add_hyperparameter(cs, weight_decay, UniformFloatHyperparameter) return cs diff --git a/autoPyTorch/pipeline/components/setup/optimizer/RMSpropOptimizer.py b/autoPyTorch/pipeline/components/setup/optimizer/RMSpropOptimizer.py index e44bb9495..8e7f0020a 100644 --- a/autoPyTorch/pipeline/components/setup/optimizer/RMSpropOptimizer.py +++ b/autoPyTorch/pipeline/components/setup/optimizer/RMSpropOptimizer.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple +from typing import Any, Dict, Optional from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( @@ -10,6 +10,7 @@ from torch.optim import RMSprop from autoPyTorch.pipeline.components.setup.optimizer.base_optimizer import BaseOptimizerComponent +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter class RMSpropOptimizer(BaseOptimizerComponent): @@ -34,7 +35,6 @@ def __init__( weight_decay: float, random_state: Optional[np.random.RandomState] = None, ): - super().__init__() self.lr = lr self.momentum = momentum @@ -76,28 +76,28 @@ def get_properties(dataset_properties: Optional[Dict[str, Any]] = None) -> Dict[ } @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, - lr: Tuple[Tuple, float, bool] = ((1e-5, 1e-1), 1e-2, True), - alpha: Tuple[Tuple, float] = ((0.1, 0.99), 0.99), - weight_decay: Tuple[Tuple, float] = ((0.0, 0.1), 0.0), - momentum: Tuple[Tuple, float] = ((0.0, 0.99), 0.0), - ) -> ConfigurationSpace: - + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict] = None, + lr: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="lr", + value_range=(1e-5, 1e-1), + default_value=1e-2, + log=True), + alpha: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="alpha", + value_range=(0.1, 0.99), + default_value=0.99), + weight_decay: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="weight_decay", + value_range=(0.0, 0.1), + default_value=0.0), + momentum: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="momentum", + value_range=(0.0, 0.99), + default_value=0.0), + ) -> ConfigurationSpace: cs = ConfigurationSpace() # The learning rate for the model - lr = UniformFloatHyperparameter('lr', lower=lr[0][0], upper=lr[0][1], - default_value=lr[1], log=lr[2]) - - alpha = UniformFloatHyperparameter('alpha', lower=alpha[0][0], upper=alpha[0][1], - default_value=alpha[1]) - - weight_decay = UniformFloatHyperparameter('weight_decay', lower=weight_decay[0][0], upper=weight_decay[0][1], - default_value=weight_decay[1]) - - momentum = UniformFloatHyperparameter('momentum', lower=momentum[0][0], upper=momentum[0][1], - default_value=momentum[1]) - - cs.add_hyperparameters([lr, alpha, weight_decay, momentum]) + add_hyperparameter(cs, lr, UniformFloatHyperparameter) + add_hyperparameter(cs, alpha, UniformFloatHyperparameter) + add_hyperparameter(cs, momentum, UniformFloatHyperparameter) + add_hyperparameter(cs, weight_decay, UniformFloatHyperparameter) return cs diff --git a/autoPyTorch/pipeline/components/setup/optimizer/SGDOptimizer.py b/autoPyTorch/pipeline/components/setup/optimizer/SGDOptimizer.py index 4396cb381..96356145b 100644 --- a/autoPyTorch/pipeline/components/setup/optimizer/SGDOptimizer.py +++ b/autoPyTorch/pipeline/components/setup/optimizer/SGDOptimizer.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple +from typing import Any, Dict, Optional from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( @@ -10,6 +10,7 @@ from torch.optim import SGD from autoPyTorch.pipeline.components.setup.optimizer.base_optimizer import BaseOptimizerComponent +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter class SGDOptimizer(BaseOptimizerComponent): @@ -70,24 +71,25 @@ def get_properties(dataset_properties: Optional[Dict[str, Any]] = None) -> Dict[ } @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, - lr: Tuple[Tuple, float, bool] = ((1e-5, 1e-1), 1e-2, True), - weight_decay: Tuple[Tuple, float] = ((0.0, 0.1), 0.0), - momentum: Tuple[Tuple, float] = ((0.0, 0.99), 0.0), - ) -> ConfigurationSpace: + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict] = None, + lr: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="lr", + value_range=(1e-5, 1e-1), + default_value=1e-2, + log=True), + weight_decay: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="weight_decay", + value_range=(0.0, 0.1), + default_value=0.0), + momentum: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="momentum", + value_range=(0.0, 0.99), + default_value=0.0), + ) -> ConfigurationSpace: cs = ConfigurationSpace() # The learning rate for the model - lr = UniformFloatHyperparameter('lr', lower=lr[0][0], upper=lr[0][1], - default_value=lr[1], log=lr[2]) - - weight_decay = UniformFloatHyperparameter('weight_decay', lower=weight_decay[0][0], upper=weight_decay[0][1], - default_value=weight_decay[1]) - - momentum = UniformFloatHyperparameter('momentum', lower=momentum[0][0], upper=momentum[0][1], - default_value=momentum[1]) - - cs.add_hyperparameters([lr, weight_decay, momentum]) + add_hyperparameter(cs, lr, UniformFloatHyperparameter) + add_hyperparameter(cs, momentum, UniformFloatHyperparameter) + add_hyperparameter(cs, weight_decay, UniformFloatHyperparameter) return cs diff --git a/autoPyTorch/pipeline/components/setup/optimizer/base_optimizer_choice.py b/autoPyTorch/pipeline/components/setup/optimizer/base_optimizer_choice.py index 5196f0bb7..93f61e74b 100644 --- a/autoPyTorch/pipeline/components/setup/optimizer/base_optimizer_choice.py +++ b/autoPyTorch/pipeline/components/setup/optimizer/base_optimizer_choice.py @@ -143,14 +143,25 @@ def get_hyperparameter_search_space( if default_ in available_optimizer: default = default_ break - - optimizer = CSH.CategoricalHyperparameter( - '__choice__', - list(available_optimizer.keys()), - default_value=default - ) + updates = self._get_search_space_updates() + if '__choice__' in updates.keys(): + choice_hyperparameter = updates['__choice__'] + if not set(choice_hyperparameter.value_range).issubset(available_optimizer): + raise ValueError("Expected given update for {} to have " + "choices in {} got {}".format(self.__class__.__name__, + available_optimizer, + choice_hyperparameter.value_range)) + optimizer = CSH.CategoricalHyperparameter('__choice__', + choice_hyperparameter.value_range, + default_value=choice_hyperparameter.default_value) + else: + optimizer = CSH.CategoricalHyperparameter( + '__choice__', + list(available_optimizer.keys()), + default_value=default + ) cs.add_hyperparameter(optimizer) - for name in available_optimizer: + for name in optimizer.choices: updates = self._get_search_space_updates(prefix=name) config_space = available_optimizer[name].get_hyperparameter_search_space(dataset_properties, # type: ignore **updates) diff --git a/autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py b/autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py index 95a4fccb2..afb43ea97 100644 --- a/autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py +++ b/autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple +from typing import Any, Dict, Optional from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import ( @@ -11,11 +11,15 @@ import torchvision - from autoPyTorch.datasets.base_dataset import BaseDataset from autoPyTorch.pipeline.components.training.base_training import autoPyTorchTrainingComponent from autoPyTorch.utils.backend import Backend -from autoPyTorch.utils.common import FitRequirement, custom_collate_fn +from autoPyTorch.utils.common import ( + FitRequirement, + HyperparameterSearchSpace, + add_hyperparameter, + custom_collate_fn +) class BaseDataLoaderComponent(autoPyTorchTrainingComponent): @@ -201,10 +205,7 @@ def check_requirements(self, X: Dict[str, Any], y: Any = None) -> None: # or from X, Y pairs if 'split_id' not in X: raise ValueError("To fit a data loader, expected fit dictionary to have split_id. " - "Currently X={}.".format( - X - ) - ) + "Currently X={}.".format(X)) if 'backend' not in X: raise ValueError("backend is needed to load the data from disk") @@ -248,13 +249,14 @@ def get_torchvision_datasets(self) -> Dict[str, torchvision.datasets.VisionDatas } @staticmethod - def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, - batch_size: Tuple[Tuple, int] = ((32, 320), 64) - ) -> ConfigurationSpace: - batch_size = UniformIntegerHyperparameter( - "batch_size", batch_size[0][0], batch_size[0][1], default_value=batch_size[1]) + def get_hyperparameter_search_space( + dataset_properties: Optional[Dict] = None, + batch_size: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="batch_size", + value_range=(32, 320), + default_value=64) + ) -> ConfigurationSpace: cs = ConfigurationSpace() - cs.add_hyperparameters([batch_size]) + add_hyperparameter(cs, batch_size, UniformIntegerHyperparameter) return cs def __str__(self) -> str: diff --git a/autoPyTorch/pipeline/components/training/trainer/MixUpTrainer.py b/autoPyTorch/pipeline/components/training/trainer/MixUpTrainer.py index ef31c27c5..f001689ea 100644 --- a/autoPyTorch/pipeline/components/training/trainer/MixUpTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/MixUpTrainer.py @@ -12,6 +12,7 @@ from autoPyTorch.constants import CLASSIFICATION_TASKS, STRING_TO_TASK_TYPES from autoPyTorch.pipeline.components.training.trainer.base_trainer import BaseTrainerComponent +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter class MixUpTrainer(BaseTrainerComponent): @@ -65,19 +66,19 @@ def get_properties(dataset_properties: typing.Optional[typing.Dict[str, typing.A } @staticmethod - def get_hyperparameter_search_space(dataset_properties: typing.Optional[typing.Dict] = None, - alpha: typing.Tuple[typing.Tuple[float, float], float] = ((0, 1), 0.2), - weighted_loss: typing.Tuple[typing.Tuple, bool] = ((True, False), True) - ) -> ConfigurationSpace: - alpha = UniformFloatHyperparameter( - "alpha", alpha[0][0], alpha[0][1], default_value=alpha[1]) + def get_hyperparameter_search_space( + dataset_properties: typing.Optional[typing.Dict] = None, + alpha: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="alpha", + value_range=(0, 1), + default_value=0.2), + weighted_loss: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="weighted_loss", + value_range=(True, False), + default_value=True), + ) -> ConfigurationSpace: cs = ConfigurationSpace() - cs.add_hyperparameters([alpha]) + add_hyperparameter(cs, alpha, UniformFloatHyperparameter) if dataset_properties is not None: if STRING_TO_TASK_TYPES[dataset_properties['task_type']] in CLASSIFICATION_TASKS: - weighted_loss = CategoricalHyperparameter("weighted_loss", - choices=weighted_loss[0], - default_value=weighted_loss[1]) - cs.add_hyperparameter(weighted_loss) + add_hyperparameter(cs, weighted_loss, CategoricalHyperparameter) return cs diff --git a/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py b/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py index 6acfb2982..e3301a2d2 100644 --- a/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py @@ -7,6 +7,7 @@ from autoPyTorch.constants import CLASSIFICATION_TASKS, STRING_TO_TASK_TYPES from autoPyTorch.pipeline.components.training.trainer.base_trainer import BaseTrainerComponent +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter class StandardTrainer(BaseTrainerComponent): @@ -53,15 +54,15 @@ def get_properties(dataset_properties: typing.Optional[typing.Dict[str, typing.A } @staticmethod - def get_hyperparameter_search_space(dataset_properties: typing.Optional[typing.Dict] = None, - weighted_loss: typing.Tuple[typing.Tuple, bool] = ((True, False), True) - ) -> ConfigurationSpace: + def get_hyperparameter_search_space( + dataset_properties: typing.Optional[typing.Dict] = None, + weighted_loss: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="weighted_loss", + value_range=(True, False), + default_value=True), + ) -> ConfigurationSpace: cs = ConfigurationSpace() if dataset_properties is not None: if STRING_TO_TASK_TYPES[dataset_properties['task_type']] in CLASSIFICATION_TASKS: - weighted_loss = CategoricalHyperparameter("weighted_loss", - choices=weighted_loss[0], - default_value=weighted_loss[1]) - cs.add_hyperparameter(weighted_loss) + add_hyperparameter(cs, weighted_loss, CategoricalHyperparameter) return cs diff --git a/autoPyTorch/pipeline/components/training/trainer/base_trainer_choice.py b/autoPyTorch/pipeline/components/training/trainer/base_trainer_choice.py index 564d199dd..248d8085b 100755 --- a/autoPyTorch/pipeline/components/training/trainer/base_trainer_choice.py +++ b/autoPyTorch/pipeline/components/training/trainer/base_trainer_choice.py @@ -137,14 +137,25 @@ def get_hyperparameter_search_space( if default_ in available_trainers: default = default_ break - - trainer = CategoricalHyperparameter( - '__choice__', - list(available_trainers.keys()), - default_value=default - ) + updates = self._get_search_space_updates() + if '__choice__' in updates.keys(): + choice_hyperparameter = updates['__choice__'] + if not set(choice_hyperparameter.value_range).issubset(available_trainers): + raise ValueError("Expected given update for {} to have " + "choices in {} got {}".format(self.__class__.__name__, + available_trainers, + choice_hyperparameter.value_range)) + trainer = CategoricalHyperparameter('__choice__', + choice_hyperparameter.value_range, + default_value=choice_hyperparameter.default_value) + else: + trainer = CategoricalHyperparameter( + '__choice__', + list(available_trainers.keys()), + default_value=default + ) cs.add_hyperparameter(trainer) - for name in available_trainers: + for name in trainer.choices: updates = self._get_search_space_updates(prefix=name) config_space = available_trainers[name].get_hyperparameter_search_space(dataset_properties, # type:ignore **updates) diff --git a/autoPyTorch/pipeline/tabular_classification.py b/autoPyTorch/pipeline/tabular_classification.py index 73dca2878..1ca0635b6 100644 --- a/autoPyTorch/pipeline/tabular_classification.py +++ b/autoPyTorch/pipeline/tabular_classification.py @@ -189,16 +189,14 @@ def _get_hyperparameter_search_space(self, cs=cs, dataset_properties=dataset_properties, exclude=exclude, include=include, pipeline=self.steps) - # Here we add custom code, like this with this - # is not a valid configuration + # Here we add custom code, that is used to ensure valid configurations, For example # Learned Entity Embedding is only valid when encoder is one hot encoder if 'network_embedding' in self.named_steps.keys() and 'encoder' in self.named_steps.keys(): embeddings = cs.get_hyperparameter('network_embedding:__choice__').choices if 'LearnedEntityEmbedding' in embeddings: encoders = cs.get_hyperparameter('encoder:__choice__').choices - default = cs.get_hyperparameter('network_embedding:__choice__').default_value possible_default_embeddings = copy.copy(list(embeddings)) - del possible_default_embeddings[possible_default_embeddings.index(default)] + del possible_default_embeddings[possible_default_embeddings.index('LearnedEntityEmbedding')] for encoder in encoders: if encoder == 'OneHotEncoder': diff --git a/autoPyTorch/utils/common.py b/autoPyTorch/utils/common.py index 1fa1c0f8f..98bd20a68 100644 --- a/autoPyTorch/utils/common.py +++ b/autoPyTorch/utils/common.py @@ -1,5 +1,14 @@ import hashlib -from typing import Any, Dict, Iterable, List, NamedTuple, Optional, Type, Union +from typing import Any, Dict, Iterable, List, NamedTuple, Optional, Sequence, Type, Union + +from ConfigSpace.configuration_space import ConfigurationSpace +from ConfigSpace.hyperparameters import ( + CategoricalHyperparameter, + Constant, + Hyperparameter, + UniformFloatHyperparameter, + UniformIntegerHyperparameter, +) import numpy as np @@ -10,16 +19,20 @@ import torch from torch.utils.data.dataloader import default_collate +HyperparameterValueType = Union[int, str, float] + class FitRequirement(NamedTuple): """ - A class that holds inputs required to fit a pipeline. Also indicates wether + A class that holds inputs required to fit a pipeline. Also indicates whether requirements have to be user specified or are generated by the pipeline itself. Attributes: - name: The name of the variable expected in the input dictionary - supported_types: An iterable of all types that are supported - user_defined: If false, this requirement does not have to be given to the pipeline + name (str): The name of the variable expected in the input dictionary + supported_types (Iterable[Type]): An iterable of all types that are supported + user_defined (bool): If false, this requirement does not have to be given to the pipeline + dataset_property (bool): If True, this requirement is automatically inferred + by the Dataset class """ name: str @@ -35,6 +48,34 @@ def __str__(self) -> str: self.name, self.supported_types, self.user_defined, self.dataset_property) +class HyperparameterSearchSpace(NamedTuple): + """ + A class that holds the search space for an individual hyperparameter. + Attributes: + hyperparameter (str): + name of the hyperparameter + value_range (Sequence[HyperparameterValueType]): + range of the hyperparameter, can be defined as min and + max values for Numerical hyperparameter or a list of + choices for a Categorical hyperparameter + default_value (HyperparameterValueType): + default value of the hyperparameter + log (bool): + whether to sample hyperparameter on a log scale + """ + hyperparameter: str + value_range: Sequence[HyperparameterValueType] + default_value: HyperparameterValueType + log: bool = False + + def __str__(self) -> str: + """ + String representation for the Search Space + """ + return "Hyperparameter: %s | Range: %s | Default: %s | log: %s" % ( + self.hyperparameter, self.value_range, self.default_value, self.log) + + def replace_prefix_in_config_dict(config: Dict[str, Any], prefix: str, replace: str = "") -> Dict[str, Any]: """ Replace the prefix in all keys with the specified replacement string (the empty string by @@ -156,3 +197,67 @@ def subsampler(data: Union[np.ndarray, pd.DataFrame, scipy.sparse.csr_matrix], x: Union[np.ndarray, List[int]] ) -> Union[np.ndarray, pd.DataFrame, scipy.sparse.csr_matrix]: return data[x] if isinstance(data, (np.ndarray, scipy.sparse.csr_matrix)) else data.iloc[x] + + +def get_hyperparameter(hyperparameter: HyperparameterSearchSpace, + hyperparameter_type: Type[Hyperparameter]) -> Hyperparameter: + """ + Given a hyperparameter search space, return a ConfigSpace Hyperparameter + Args: + hyperparameter (HyperparameterSearchSpace): + the search space for the hyperparameter + hyperparameter_type (Hyperparameter): + the type of the hyperparameter + + Returns: + Hyperparameter + """ + if len(hyperparameter.value_range) == 0: + raise ValueError(hyperparameter.hyperparameter + ': The range has to contain at least one element') + if len(hyperparameter.value_range) == 1 and hyperparameter_type != CategoricalHyperparameter: + return Constant(hyperparameter.hyperparameter, hyperparameter.value_range[0]) + if len(hyperparameter.value_range) == 2 and hyperparameter.value_range[0] == hyperparameter.value_range[1]: + return Constant(hyperparameter.hyperparameter, hyperparameter.value_range[0]) + if hyperparameter_type == CategoricalHyperparameter: + return CategoricalHyperparameter(hyperparameter.hyperparameter, + choices=hyperparameter.value_range, + default_value=hyperparameter.default_value) + if hyperparameter_type == UniformFloatHyperparameter: + assert len(hyperparameter.value_range) == 2, \ + "Float HP range update for %s is specified by the two upper " \ + "and lower values. %s given." % (hyperparameter.hyperparameter, len(hyperparameter.value_range)) + return UniformFloatHyperparameter(hyperparameter.hyperparameter, + lower=hyperparameter.value_range[0], + upper=hyperparameter.value_range[1], + log=hyperparameter.log, + default_value=hyperparameter.default_value) + if hyperparameter_type == UniformIntegerHyperparameter: + assert len(hyperparameter.value_range) == 2, \ + "Int HP range update for %s is specified by the two upper " \ + "and lower values. %s given." % (hyperparameter.hyperparameter, len(hyperparameter.value_range)) + return UniformIntegerHyperparameter(hyperparameter.hyperparameter, + lower=hyperparameter.value_range[0], + upper=hyperparameter.value_range[1], + log=hyperparameter.log, + default_value=hyperparameter.default_value) + raise ValueError('Unknown type: %s for hp %s' % (hyperparameter_type, hyperparameter.hyperparameter)) + + +def add_hyperparameter(cs: ConfigurationSpace, + hyperparameter: HyperparameterSearchSpace, + hyperparameter_type: Type[Hyperparameter]) -> None: + """ + Adds the given hyperparameter to the given configuration space + + Args: + cs (ConfigurationSpace): + Configuration space where the hyperparameter must be added + hyperparameter (HyperparameterSearchSpace): + search space of the hyperparameter + hyperparameter_type (Hyperparameter): + type of the hyperparameter + + Returns: + None + """ + return cs.add_hyperparameter(get_hyperparameter(hyperparameter, hyperparameter_type)) diff --git a/autoPyTorch/utils/hyperparameter_search_space_update.py b/autoPyTorch/utils/hyperparameter_search_space_update.py index e2ef1c85f..9891f5f45 100644 --- a/autoPyTorch/utils/hyperparameter_search_space_update.py +++ b/autoPyTorch/utils/hyperparameter_search_space_update.py @@ -1,9 +1,10 @@ import ast import os -from typing import List, Optional, Tuple, Union +from typing import List, Optional, Sequence, Tuple, Union -from autoPyTorch.pipeline.components.base_choice import autoPyTorchChoice -from autoPyTorch.pipeline.components.base_component import autoPyTorchComponent +from sklearn.base import BaseEstimator + +from autoPyTorch.utils.common import HyperparameterSearchSpace, HyperparameterValueType class HyperparameterSearchSpaceUpdate: @@ -16,38 +17,36 @@ class HyperparameterSearchSpaceUpdate: The name of the node in the pipeline hyperparameter (str): The name of the hyperparameter - value_range (Union[List, Tuple]): + value_range (Sequence[HyperparameterValueType]): In case of categorical hyperparameter, defines the new categorical choices. In case of numerical hyperparameter, defines the new range in the form of (LOWER, UPPER) - default_value (Union[int, float, str]): + default_value (HyperparameterValueType): New default value for the hyperparameter log (bool) (default=False): In case of numerical hyperparameters, whether to sample on a log scale """ - def __init__(self, node_name: str, hyperparameter: str, value_range: Union[List, Tuple], - default_value: Union[int, float, str], log: bool = False) -> None: + def __init__(self, node_name: str, hyperparameter: str, value_range: Sequence[HyperparameterValueType], + default_value: HyperparameterValueType, log: bool = False) -> None: self.node_name = node_name self.hyperparameter = hyperparameter + if len(value_range) == 0: + raise ValueError("The new value range needs at least one value") self.value_range = value_range self.log = log self.default_value = default_value - def apply(self, pipeline: List[Tuple[str, Union[autoPyTorchComponent, autoPyTorchChoice]]]) -> None: + def apply(self, pipeline: List[Tuple[str, BaseEstimator]]) -> None: """ Applies the update to the appropriate hyperparameter of the pipeline Args: - pipeline (List[Tuple[str, Union[autoPyTorchComponent, autoPyTorchChoice]]]): + pipeline (List[Tuple[str, BaseEstimator]]): The named steps of the current autopytorch pipeline Returns: None """ - [node[1]._apply_search_space_update(name=self.hyperparameter, - new_value_range=self.value_range, - log=self.log, - default_value=self.default_value) - for node in pipeline if node[0] == self.node_name] + [node[1]._apply_search_space_update(self) for node in pipeline if node[0] == self.node_name] def __str__(self) -> str: return "{}, {}, {}, {}, {}".format(self.node_name, self.hyperparameter, str(self.value_range), @@ -55,18 +54,39 @@ def __str__(self) -> str: str) else self.default_value, (" log" if self.log else "")) + def get_search_space(self, remove_prefix: Optional[str] = None) -> HyperparameterSearchSpace: + """ + Get Update as a HyperparameterSearchSpace object. + + Args: + remove_prefix (Optional[str]): + if specified, remove given prefix from hyperparameter name + + Returns: + HyperparameterSearchSpace + """ + hyperparameter_name = self.hyperparameter + if remove_prefix is not None: + # remove prefix from hyperparameter name + if remove_prefix in self.hyperparameter: + hyperparameter_name = hyperparameter_name.replace(f"{remove_prefix}:", '') + return HyperparameterSearchSpace(hyperparameter=hyperparameter_name, + value_range=self.value_range, + default_value=self.default_value, + log=self.log) + class HyperparameterSearchSpaceUpdates: """ Contains a collection of HyperparameterSearchSpaceUpdate """ def __init__(self, updates: Optional[List[HyperparameterSearchSpaceUpdate]] = None) -> None: self.updates = updates if updates is not None else [] - def apply(self, pipeline: List[Tuple[str, Union[autoPyTorchComponent, autoPyTorchChoice]]]) -> None: + def apply(self, pipeline: List[Tuple[str, BaseEstimator]]) -> None: """ Iteratively applies updates to the pipeline Args: - pipeline: (List[Tuple[str, Union[autoPyTorchComponent, autoPyTorchChoice]]]): + pipeline: (List[Tuple[str, BaseEstimator]]): The named steps of the current autoPyTorch pipeline Returns: diff --git a/test/conftest.py b/test/conftest.py index 04077ca08..06f18e0f9 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -259,7 +259,9 @@ def get_tabular_data(task): y = y.iloc[0:200] y = (y - y.mean()) / y.std() validator = TabularInputValidator(is_classification=False).fit(X.copy(), y.copy()) - + elif task == 'iris': + X, y = fetch_openml("iris", return_X_y=True, as_frame=True) + validator = TabularInputValidator(is_classification=True).fit(X.copy(), y.copy()) else: raise ValueError("Unsupported task {}".format(task)) diff --git a/test/test_pipeline/components/setup/test_setup.py b/test/test_pipeline/components/setup/test_setup.py index 9349c4ac8..aae1c8ff4 100644 --- a/test/test_pipeline/components/setup/test_setup.py +++ b/test/test_pipeline/components/setup/test_setup.py @@ -309,7 +309,7 @@ def test_dummy_forward_backward_pass(self, task_type_input_shape): value_range=[1, 3], default_value=2) updates.append(node_name='network_backbone', - hyperparameter='ConvNetImageBackbone:num_init_filters', + hyperparameter='ConvNetImageBackbone:conv_init_filters', value_range=[8, 16], default_value=8) updates.append(node_name='network_backbone', diff --git a/test/test_pipeline/components/setup/test_setup_networks.py b/test/test_pipeline/components/setup/test_setup_networks.py index f6f3decb0..6826d7ef2 100644 --- a/test/test_pipeline/components/setup/test_setup_networks.py +++ b/test/test_pipeline/components/setup/test_setup_networks.py @@ -47,8 +47,6 @@ def test_pipeline_fit(self, fit_dictionary_tabular, embedding, backbone, head): assert backbone == config.get('network_backbone:__choice__', None) assert head == config.get('network_head:__choice__', None) pipeline.set_hyperparameters(config) - # Need more epochs to make sure validation performance is met - fit_dictionary_tabular['epochs'] = 100 # Early stop to the best configuration seen fit_dictionary_tabular['early_stopping'] = 50 diff --git a/test/test_pipeline/test_tabular_classification.py b/test/test_pipeline/test_tabular_classification.py index ef508dc7b..108a217e7 100644 --- a/test/test_pipeline/test_tabular_classification.py +++ b/test/test_pipeline/test_tabular_classification.py @@ -330,4 +330,111 @@ def test_set_range_search_space_updates(self, fit_dictionary_tabular): except AssertionError as e: # As we are setting num_layers to 1 for fully connected # head, units_layer does not exist in the configspace - assert 'fully_connected:units_layer' in e.args[0] + assert 'fully_connected:units_layer' in e.args[0], e.args[0] + + def test_set_choices_updates(self, fit_dictionary_tabular): + dataset_properties = {'numerical_columns': [1], 'categorical_columns': [2], + 'task_type': 'tabular_classification'} + config_dict = TabularClassificationPipeline(dataset_properties=dataset_properties). \ + get_hyperparameter_search_space()._hyperparameters + updates = HyperparameterSearchSpaceUpdates() + for i, (name, hyperparameter) in enumerate(config_dict.items()): + if '__choice__' not in name: + continue + name = name.split(':') + hyperparameter_name = ':'.join(name[1:]) + # Using NoEmbedding is safer for this test + # to avoid forbidden configuration errors + if name[0] == 'network_embedding' and hyperparameter_name == '__choice__': + value_range = ('NoEmbedding',) + default_value = 'NoEmbedding' + else: + value_range = (hyperparameter.choices[0],) + default_value = hyperparameter.choices[0] + updates.append(node_name=name[0], hyperparameter=hyperparameter_name, + value_range=value_range, default_value=default_value) + pipeline = TabularClassificationPipeline(dataset_properties=dataset_properties, + search_space_updates=updates) + self._assert_pipeline_search_space(pipeline, updates) + + +@pytest.mark.parametrize("fit_dictionary_tabular", ['iris'], indirect=True) +def test_constant_pipeline_iris(fit_dictionary_tabular): + search_space_updates = HyperparameterSearchSpaceUpdates() + search_space_updates.append(node_name='feature_preprocessor', + hyperparameter='__choice__', + value_range=['PolynomialFeatures'], + default_value='PolynomialFeatures') + search_space_updates.append(node_name='scaler', + hyperparameter='__choice__', + value_range=['StandardScaler'], + default_value='StandardScaler') + search_space_updates.append(node_name='network_backbone', + hyperparameter='__choice__', + value_range=['MLPBackbone', 'ShapedMLPBackbone'], + default_value='MLPBackbone') + search_space_updates.append(node_name='network_backbone', + hyperparameter='MLPBackbone:num_groups', + value_range=[1, 1], + default_value=1) + search_space_updates.append(node_name='network_backbone', + hyperparameter='MLPBackbone:num_units', + value_range=[100], + default_value=100) + search_space_updates.append(node_name='trainer', + hyperparameter='__choice__', + value_range=['StandardTrainer'], + default_value='StandardTrainer') + search_space_updates.append(node_name='lr_scheduler', + hyperparameter='__choice__', + value_range=['NoScheduler'], + default_value='NoScheduler') + search_space_updates.append(node_name='optimizer', + hyperparameter='__choice__', + value_range=['AdamOptimizer'], + default_value='AdamOptimizer') + search_space_updates.append(node_name='optimizer', + hyperparameter='AdamOptimizer:lr', + value_range=[1e-2], + default_value=1e-2) + pipeline = TabularClassificationPipeline(dataset_properties=fit_dictionary_tabular['dataset_properties'], + search_space_updates=search_space_updates) + + fit_dictionary_tabular['additional_metrics'] = ['balanced_accuracy'] + + try: + pipeline.fit(fit_dictionary_tabular) + except Exception as e: + pytest.fail(f"Failed due to {e}") + + configuration = pipeline.configuration + + assert 'PolynomialFeatures' == configuration.get('feature_preprocessor:__choice__') + assert 'StandardScaler' == configuration.get('scaler:__choice__') + assert 'MLPBackbone' == configuration.get('network_backbone:__choice__') + assert 'StandardTrainer' == configuration.get('trainer:__choice__') + assert 'NoScheduler' == configuration.get('lr_scheduler:__choice__') + assert 'AdamOptimizer' == configuration.get('optimizer:__choice__') + assert 1 == configuration.get('network_backbone:MLPBackbone:num_groups') + assert 100 == configuration.get('network_backbone:MLPBackbone:num_units_1') + assert 1e-2 == configuration.get('optimizer:AdamOptimizer:lr') + + # To make sure we fitted the model, there should be a + # run summary object with accuracy + run_summary = pipeline.named_steps['trainer'].run_summary + assert run_summary is not None + + # Make sure that performance was properly captured + assert run_summary.performance_tracker['train_loss'][1] > 0 + assert run_summary.total_parameter_count > 0 + assert 'balanced_accuracy' in run_summary.performance_tracker['train_metrics'][1] + + # Make sure default pipeline achieves a good score for dummy datasets + epoch2loss = run_summary.performance_tracker['val_loss'] + best_loss = min(list(epoch2loss.values())) + epoch_where_best = list(epoch2loss.keys())[list(epoch2loss.values()).index(best_loss)] + val_score = run_summary.performance_tracker['val_metrics'][epoch_where_best]['balanced_accuracy'] + train_score = run_summary.performance_tracker['train_metrics'][epoch_where_best]['balanced_accuracy'] + + assert val_score >= 0.9, run_summary.performance_tracker['val_metrics'] + assert train_score >= 0.9, run_summary.performance_tracker['train_metrics'] diff --git a/test/test_utils/test_hyperparameter_search_space_update.py b/test/test_utils/test_hyperparameter_search_space_update.py new file mode 100644 index 000000000..ee4af4814 --- /dev/null +++ b/test/test_utils/test_hyperparameter_search_space_update.py @@ -0,0 +1,84 @@ +from ConfigSpace.configuration_space import ConfigurationSpace +from ConfigSpace.hyperparameters import ( + CategoricalHyperparameter, + UniformFloatHyperparameter, + UniformIntegerHyperparameter +) + +from autoPyTorch.pipeline.components.base_component import autoPyTorchComponent +from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter +from autoPyTorch.utils.hyperparameter_search_space_update import HyperparameterSearchSpaceUpdates + + +class DummyComponent(autoPyTorchComponent): + def __init__(self): + self._cs_updates = {} + + @staticmethod + def get_hyperparameter_search_space( + dataset_properties=None, + X=HyperparameterSearchSpace("X", + value_range=[-5, 5], + default_value=0), + Y=HyperparameterSearchSpace("Y", + value_range=[0, 1], + default_value=0), + Z=HyperparameterSearchSpace("Z", + value_range=['a', 'b', 1], + default_value='a'), + ): + cs = ConfigurationSpace() + add_hyperparameter(cs, X, UniformIntegerHyperparameter) + add_hyperparameter(cs, Y, UniformFloatHyperparameter) + add_hyperparameter(cs, Z, CategoricalHyperparameter) + return cs + + +def test_hyperparameter_search_space_update(): + updates = HyperparameterSearchSpaceUpdates() + updates.append(node_name="dummy_node", + hyperparameter="X", + value_range=[1, 3], + default_value=2) + updates.append(node_name="dummy_node", + hyperparameter="Y", + value_range=[0.1, 0.5], + default_value=0.1, + log=True) + updates.append(node_name="dummy_node", + hyperparameter="Z", + value_range=['a', 3], + default_value=3) + dummy_component = DummyComponent() + updates.apply([("dummy_node", dummy_component)]) + new_updates = dummy_component._get_search_space_updates() + config_space = dummy_component.get_hyperparameter_search_space(**new_updates) + + for i, (update, hp_type) in enumerate(zip(['X', 'Y', 'Z'], + [UniformIntegerHyperparameter, + UniformFloatHyperparameter, + CategoricalHyperparameter])): + + search_space_update = updates.updates[i] + search_space = search_space_update.get_search_space() + + assert search_space.hyperparameter == search_space_update.hyperparameter + assert search_space.value_range == search_space_update.value_range + assert search_space.default_value == search_space_update.default_value + assert search_space.log == search_space_update.log + + assert update in dummy_component._cs_updates + assert update in new_updates + assert update in config_space + + hp = config_space.get_hyperparameter(update) + assert isinstance(hp, hp_type) + + if update == 'Z': + assert all(a == b for a, b in zip(hp.choices, search_space.value_range)) + else: + assert hp.lower == search_space.value_range[0] + assert hp.upper == search_space.value_range[1] + assert hp.log == search_space.log + + assert hp.default_value == search_space.default_value