From 01ccf5dfd914536b306d468f35a285ea95bc389d Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Tue, 13 Apr 2021 19:25:45 +0200 Subject: [PATCH 01/16] In progress get_incumbent_results --- autoPyTorch/api/base_task.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/autoPyTorch/api/base_task.py b/autoPyTorch/api/base_task.py index dd26ebf99..dd8a1ed38 100644 --- a/autoPyTorch/api/base_task.py +++ b/autoPyTorch/api/base_task.py @@ -1263,15 +1263,17 @@ def __del__(self) -> None: @typing.no_type_check def get_incumbent_results( - self - ): - pass - - @typing.no_type_check - def get_incumbent_config( - self + self, + include_traditional: bool = False ): - pass + assert self.run_history is not None, "No Run History found, search has not been called." + assert not self.run_history.empty(), "Run History is empty." + + sorted_runvalue_by_cost = sorted(self.run_history.data.items(), key=lambda item: item[1].cost) + incumbent_run_key, incumbent_run_value = sorted_runvalue_by_cost[0] + incumbent_config = self.run_history.ids_config[incumbent_run_key.config_id] + incumbent_results = incumbent_run_value.additional_info + return incumbent_config, incumbent_results def get_models_with_weights(self) -> List: if self.models_ is None or len(self.models_) == 0 or \ From 241de5d087c0d26b3792d8090bb13813942b7926 Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Wed, 14 Apr 2021 10:52:41 +0200 Subject: [PATCH 02/16] [Add] get_incumbent_results to base task, changed additional info in abstract evaluator, and tests --- autoPyTorch/api/base_task.py | 63 +++++++++++--------- autoPyTorch/evaluation/abstract_evaluator.py | 3 +- test/test_api/test_api.py | 16 +++++ 3 files changed, 52 insertions(+), 30 deletions(-) diff --git a/autoPyTorch/api/base_task.py b/autoPyTorch/api/base_task.py index dd8a1ed38..1917feb8c 100644 --- a/autoPyTorch/api/base_task.py +++ b/autoPyTorch/api/base_task.py @@ -12,7 +12,7 @@ import unittest.mock import warnings from abc import abstractmethod -from typing import Any, Callable, Dict, List, Optional, Union, cast +from typing import Any, Callable, Dict, List, Optional, Tuple, Union, cast from ConfigSpace.configuration_space import Configuration, ConfigurationSpace @@ -212,9 +212,7 @@ def build_pipeline(self, dataset_properties: Dict[str, Any]) -> BasePipeline: """ raise NotImplementedError - def set_pipeline_config( - self, - **pipeline_config_kwargs: Any) -> None: + def set_pipeline_config(self, **pipeline_config_kwargs: Any) -> None: """ Check whether arguments are valid and then sets them to the current pipeline @@ -248,12 +246,6 @@ def get_pipeline_options(self) -> dict: """ return self.pipeline_options - # def set_search_space(self, search_space: ConfigurationSpace) -> None: - # """ - # Update the search space. - # """ - # raise NotImplementedError - # def get_search_space(self, dataset: BaseDataset = None) -> ConfigurationSpace: """ Returns the current search space as ConfigurationSpace object. @@ -394,9 +386,9 @@ def _close_dask_client(self) -> None: None """ if ( - hasattr(self, '_is_dask_client_internally_created') - and self._is_dask_client_internally_created - and self._dask_client + hasattr(self, '_is_dask_client_internally_created') + and self._is_dask_client_internally_created + and self._dask_client ): self._dask_client.shutdown() self._dask_client.close() @@ -647,10 +639,12 @@ def _do_traditional_prediction(self, time_left: int, func_eval_time_limit_secs: f"Fitting {cls} took {runtime}s, performance:{cost}/{additional_info}") configuration = additional_info['pipeline_configuration'] origin = additional_info['configuration_origin'] + additional_info.pop('pipeline_configuration') + additional_info.pop('configuration_origin') run_history.add(config=configuration, cost=cost, time=runtime, status=status, seed=self.seed, starttime=starttime, endtime=starttime + runtime, - origin=origin) + origin=origin, additional_info=additional_info) else: if additional_info.get('exitcode') == -6: self._logger.error( @@ -1026,10 +1020,10 @@ def _search( return self def refit( - self, - dataset: BaseDataset, - budget_config: Dict[str, Union[int, str]] = {}, - split_id: int = 0 + self, + dataset: BaseDataset, + budget_config: Dict[str, Union[int, str]] = {}, + split_id: int = 0 ) -> "BaseTask": """ Refit all models found with fit to new data. @@ -1165,10 +1159,10 @@ def fit(self, return pipeline def predict( - self, - X_test: np.ndarray, - batch_size: Optional[int] = None, - n_jobs: int = 1 + self, + X_test: np.ndarray, + batch_size: Optional[int] = None, + n_jobs: int = 1 ) -> np.ndarray: """Generate the estimator predictions. Generate the predictions based on the given examples from the test set. @@ -1218,9 +1212,9 @@ def predict( return predictions def score( - self, - y_pred: np.ndarray, - y_test: Union[np.ndarray, pd.DataFrame] + self, + y_pred: np.ndarray, + y_test: Union[np.ndarray, pd.DataFrame] ) -> Dict[str, float]: """Calculate the score on the test set. Calculate the evaluation measure on the test set. @@ -1261,15 +1255,28 @@ def __del__(self) -> None: if hasattr(self, '_backend'): self._backend.context.delete_directories(force=False) - @typing.no_type_check def get_incumbent_results( self, include_traditional: bool = False - ): + ) -> Tuple[Configuration, Dict]: + """ + Get Incumbent config and the corresponding results + Args: + include_traditional: + + Returns: + + """ assert self.run_history is not None, "No Run History found, search has not been called." assert not self.run_history.empty(), "Run History is empty." - sorted_runvalue_by_cost = sorted(self.run_history.data.items(), key=lambda item: item[1].cost) + run_history_data = self.run_history.data + if not include_traditional: + run_history_data = dict( + filter(lambda elem: elem[1].additional_info is not None and 'trainer_configuration' not in elem[1]. + additional_info, + run_history_data.items())) + sorted_runvalue_by_cost = sorted(run_history_data.items(), key=lambda item: item[1].cost) incumbent_run_key, incumbent_run_value = sorted_runvalue_by_cost[0] incumbent_config = self.run_history.ids_config[incumbent_run_key.config_id] incumbent_results = incumbent_run_value.additional_info diff --git a/autoPyTorch/evaluation/abstract_evaluator.py b/autoPyTorch/evaluation/abstract_evaluator.py index 9a4260e05..d98c18fc9 100644 --- a/autoPyTorch/evaluation/abstract_evaluator.py +++ b/autoPyTorch/evaluation/abstract_evaluator.py @@ -611,8 +611,7 @@ def finish_up(self, loss: Dict[str, float], train_loss: Dict[str, float], additional_run_info = ( {} if additional_run_info is None else additional_run_info ) - for metric_name, value in loss.items(): - additional_run_info[metric_name] = value + additional_run_info['opt_loss'] = loss additional_run_info['duration'] = self.duration additional_run_info['num_run'] = self.num_run if train_loss is not None: diff --git a/test/test_api/test_api.py b/test/test_api/test_api.py index 9f9d9f765..f27589a15 100644 --- a/test/test_api/test_api.py +++ b/test/test_api/test_api.py @@ -6,6 +6,8 @@ import unittest from test.test_api.utils import dummy_do_dummy_prediction, dummy_eval_function +from ConfigSpace.configuration_space import Configuration + import numpy as np import pandas as pd @@ -190,6 +192,13 @@ def test_tabular_classification(openml_id, resampling_strategy, backend, resampl score = estimator.score(y_pred, y_test) assert 'accuracy' in score + # check incumbent config and results + incumbent_config, incumbent_results = estimator.get_incumbent_results() + assert isinstance(incumbent_config, Configuration) + assert isinstance(incumbent_results, dict) + assert 'opt_loss' in incumbent_results + assert 'train_loss' in incumbent_results + # Check that we can pickle # Test pickle # This can happen on python greater than 3.6 @@ -365,6 +374,13 @@ def test_tabular_regression(openml_name, resampling_strategy, backend, resamplin score = estimator.score(y_pred, y_test) assert 'r2' in score + # check incumbent config and results + incumbent_config, incumbent_results = estimator.get_incumbent_results() + assert isinstance(incumbent_config, Configuration) + assert isinstance(incumbent_results, dict) + assert 'opt_loss' in incumbent_results + assert 'train_loss' in incumbent_results + # Check that we can pickle # Test pickle # This can happen on python greater than 3.6 From dcc2ef842c22d80f02587e647a0dd92f0f26dfc7 Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Thu, 15 Apr 2021 17:08:44 +0200 Subject: [PATCH 03/16] In progress addressing fransisco's comment --- autoPyTorch/api/base_task.py | 9 +++++--- test/test_api/test_api.py | 40 +++++++++++++++++++++++++++++++++--- 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/autoPyTorch/api/base_task.py b/autoPyTorch/api/base_task.py index 1917feb8c..65a944315 100644 --- a/autoPyTorch/api/base_task.py +++ b/autoPyTorch/api/base_task.py @@ -1258,20 +1258,23 @@ def __del__(self) -> None: def get_incumbent_results( self, include_traditional: bool = False - ) -> Tuple[Configuration, Dict]: + ) -> Tuple[Configuration, Dict[str, Union[int, str, float]]]: """ Get Incumbent config and the corresponding results Args: - include_traditional: + include_traditional: Whether to include results from tradtional pipelines Returns: """ assert self.run_history is not None, "No Run History found, search has not been called." - assert not self.run_history.empty(), "Run History is empty." + if self.run_history.empty(): + raise ValueError("Run History is empty. Something went wrong, " + "smac was not able to fit any model?") run_history_data = self.run_history.data if not include_traditional: + # traditional classifiers have trainer_configuration in their additional info run_history_data = dict( filter(lambda elem: elem[1].additional_info is not None and 'trainer_configuration' not in elem[1]. additional_info, diff --git a/test/test_api/test_api.py b/test/test_api/test_api.py index f27589a15..9c0f41776 100644 --- a/test/test_api/test_api.py +++ b/test/test_api/test_api.py @@ -30,6 +30,9 @@ ) from autoPyTorch.optimizer.smbo import AutoMLSMBO from autoPyTorch.pipeline.components.training.metrics.metrics import accuracy +======= +from autoPyTorch.evaluation.tae import ExecuteTaFuncWithQueue +>>>>>>> In progress addressing fransisco's comment CV_NUM_SPLITS = 2 @@ -551,13 +554,12 @@ def test_portfolio_selection_failure(openml_id, backend, n_samples): # is not able to be stored on disk (only on CI) if sys.version_info < (3, 7): include = {'network_embedding': ['NoEmbedding']} - # Search for a good configuration + # Search for a good configuration estimator = TabularClassificationTask( backend=backend, resampling_strategy=HoldoutValTypes.holdout_validation, - include_components=include + include_components = include ) - with pytest.raises(FileNotFoundError, match=r"The path: .+? provided for 'portfolio_selection' " r"for the file containing the portfolio configurations " r"does not exist\. Please provide a valid path"): @@ -570,3 +572,35 @@ def test_portfolio_selection_failure(openml_id, backend, n_samples): enable_traditional_pipeline=False, portfolio_selection="random_path_to_test.json" ) + + +@pytest.mark.parametrize('dataset_name', ('iris',)) +def test_get_incumbent_results(dataset_name, backend): + # Get the data and check that contents of data-manager make sense + X, y = sklearn.datasets.fetch_openml( + name=dataset_name, + return_X_y=True, as_frame=True + ) + + X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split( + X, y, random_state=1) + # Search for a good configuration + estimator = TabularClassificationTask( + backend=backend, + resampling_strategy=HoldoutValTypes.holdout_validation, + ensemble_size=0, + ) + + estimator._do_dummy_prediction = unittest.mock.MagicMock() + + # with unittest.mock.patch.object(ExecuteTaFuncWithQueue, 'run') as TAEMock: + # TAEMock.return_value = + # estimator.search( + # X_train=X_train, y_train=y_train, + # X_test=X_test, y_test=y_test, + # optimize_metric='accuracy', + # total_walltime_limit=150, + # func_eval_time_limit_secs=50, + # enable_traditional_pipeline=False, + # load_models=False, + # ) From 6bd20cd9c729c30b47d673d5d704162e09360970 Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Mon, 19 Apr 2021 15:29:25 +0200 Subject: [PATCH 04/16] Proper check for include_traditional --- autoPyTorch/api/base_task.py | 7 ++++++- test/test_api/test_api.py | 32 ++++++++++++++++---------------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/autoPyTorch/api/base_task.py b/autoPyTorch/api/base_task.py index 65a944315..cd817bf75 100644 --- a/autoPyTorch/api/base_task.py +++ b/autoPyTorch/api/base_task.py @@ -690,6 +690,7 @@ def _search( memory_limit: Optional[int] = 4096, smac_scenario_args: Optional[Dict[str, Any]] = None, get_smac_object_callback: Optional[Callable] = None, + tae_func: Optional[Callable] = None, all_supported_metrics: bool = True, precision: int = 32, disable_file_output: List = [], @@ -757,6 +758,10 @@ def _search( instances, num_params, runhistory, seed and ta. This is an advanced feature. Use only if you are familiar with [SMAC](https://automl.github.io/SMAC3/master/index.html). + tae_func (Optional[Callable]): + TargetAlgorithm to be optimised. If None, `eval_function` + available in autoPyTorch/evaluation/train_evaluator is used. + Must be child class of AbstractEvaluator. all_supported_metrics (bool), (default=True): if True, all metrics supporting current task will be calculated for each pipeline and results will be available via cv_results @@ -966,7 +971,7 @@ def _search( ) try: run_history, self.trajectory, budget_type = \ - _proc_smac.run_smbo() + _proc_smac.run_smbo(func=tae_func) self.run_history.update(run_history, DataOrigin.INTERNAL) trajectory_filename = os.path.join( self._backend.get_smac_output_directory_for_run(self.seed), diff --git a/test/test_api/test_api.py b/test/test_api/test_api.py index 9c0f41776..d13597582 100644 --- a/test/test_api/test_api.py +++ b/test/test_api/test_api.py @@ -30,9 +30,6 @@ ) from autoPyTorch.optimizer.smbo import AutoMLSMBO from autoPyTorch.pipeline.components.training.metrics.metrics import accuracy -======= -from autoPyTorch.evaluation.tae import ExecuteTaFuncWithQueue ->>>>>>> In progress addressing fransisco's comment CV_NUM_SPLITS = 2 @@ -575,7 +572,8 @@ def test_portfolio_selection_failure(openml_id, backend, n_samples): @pytest.mark.parametrize('dataset_name', ('iris',)) -def test_get_incumbent_results(dataset_name, backend): +@pytest.mark.parametrize('include_traditional', (True, False)) +def test_get_incumbent_results(dataset_name, backend, include_traditional): # Get the data and check that contents of data-manager make sense X, y = sklearn.datasets.fetch_openml( name=dataset_name, @@ -588,19 +586,21 @@ def test_get_incumbent_results(dataset_name, backend): estimator = TabularClassificationTask( backend=backend, resampling_strategy=HoldoutValTypes.holdout_validation, - ensemble_size=0, ) estimator._do_dummy_prediction = unittest.mock.MagicMock() + estimator.search( + X_train=X_train, y_train=y_train, + X_test=X_test, y_test=y_test, + optimize_metric='accuracy', + total_walltime_limit=150, + func_eval_time_limit_secs=50, + enable_traditional_pipeline=True, + load_models=False, + ) + config, results = estimator.get_incumbent_results(include_traditional=include_traditional) + assert isinstance(config, Configuration) + assert isinstance(results, dict) - # with unittest.mock.patch.object(ExecuteTaFuncWithQueue, 'run') as TAEMock: - # TAEMock.return_value = - # estimator.search( - # X_train=X_train, y_train=y_train, - # X_test=X_test, y_test=y_test, - # optimize_metric='accuracy', - # total_walltime_limit=150, - # func_eval_time_limit_secs=50, - # enable_traditional_pipeline=False, - # load_models=False, - # ) + if not include_traditional: + assert results['configuration_origin'] != 'traditional' \ No newline at end of file From c2e1c344e5a399a34442c998a983344b677867c8 Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Mon, 19 Apr 2021 18:03:21 +0200 Subject: [PATCH 05/16] Fix flake --- test/test_api/test_api.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/test_api/test_api.py b/test/test_api/test_api.py index d13597582..a921084ed 100644 --- a/test/test_api/test_api.py +++ b/test/test_api/test_api.py @@ -36,7 +36,6 @@ HOLDOUT_NUM_SPLITS = 1 -# ==== # Test # ==== @unittest.mock.patch('autoPyTorch.evaluation.train_evaluator.eval_function', @@ -603,4 +602,4 @@ def test_get_incumbent_results(dataset_name, backend, include_traditional): assert isinstance(results, dict) if not include_traditional: - assert results['configuration_origin'] != 'traditional' \ No newline at end of file + assert results['configuration_origin'] != 'traditional' From bb73f0ee819a57f22396acce2f0dca0bd07d6dc9 Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Thu, 29 Apr 2021 13:30:46 +0200 Subject: [PATCH 06/16] Mock search of estimator --- test/test_api/test_api.py | 48 +++++++++++++++++++++++++++++++-------- test/test_api/utils.py | 17 ++++++++++++++ 2 files changed, 55 insertions(+), 10 deletions(-) diff --git a/test/test_api/test_api.py b/test/test_api/test_api.py index a921084ed..13d6eaaf2 100644 --- a/test/test_api/test_api.py +++ b/test/test_api/test_api.py @@ -4,7 +4,7 @@ import pickle import sys import unittest -from test.test_api.utils import dummy_do_dummy_prediction, dummy_eval_function +from test.test_api.utils import dummy_do_dummy_prediction, dummy_eval_function, dummy_traditional_classification from ConfigSpace.configuration_space import Configuration @@ -24,10 +24,12 @@ from autoPyTorch.api.tabular_classification import TabularClassificationTask from autoPyTorch.api.tabular_regression import TabularRegressionTask +from autoPyTorch.data.tabular_validator import TabularInputValidator from autoPyTorch.datasets.resampling_strategy import ( CrossValTypes, HoldoutValTypes, ) +from autoPyTorch.datasets.tabular_dataset import TabularDataset from autoPyTorch.optimizer.smbo import AutoMLSMBO from autoPyTorch.pipeline.components.training.metrics.metrics import accuracy @@ -581,22 +583,48 @@ def test_get_incumbent_results(dataset_name, backend, include_traditional): X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split( X, y, random_state=1) + # Search for a good configuration estimator = TabularClassificationTask( backend=backend, resampling_strategy=HoldoutValTypes.holdout_validation, ) - estimator._do_dummy_prediction = unittest.mock.MagicMock() - estimator.search( - X_train=X_train, y_train=y_train, - X_test=X_test, y_test=y_test, - optimize_metric='accuracy', - total_walltime_limit=150, - func_eval_time_limit_secs=50, - enable_traditional_pipeline=True, - load_models=False, + InputValidator = TabularInputValidator( + is_classification=True, ) + + # Fit a input validator to check the provided data + # Also, an encoder is fit to both train and test data, + # to prevent unseen categories during inference + InputValidator.fit(X_train=X_train, y_train=y_train, X_test=X_test, y_test=y_test) + + dataset = TabularDataset( + X=X_train, Y=y_train, + X_test=X_test, Y_test=y_test, + validator=InputValidator, + resampling_strategy=estimator.resampling_strategy, + resampling_strategy_args=estimator.resampling_strategy_args, + ) + + pipeline_run_history = RunHistory() + pipeline_run_history.load_json('./.tmp_api/runhistory.json', estimator.get_search_space(dataset)) + + estimator._do_dummy_prediction = unittest.mock.MagicMock() + + with unittest.mock.patch.object(AutoMLSMBO, 'run_smbo') as AutoMLSMBOMock: + with unittest.mock.patch.object(TabularClassificationTask, '_do_traditional_prediction', + new=dummy_traditional_classification): + AutoMLSMBOMock.return_value = (pipeline_run_history, {}, 'epochs') + estimator.search( + X_train=X_train, y_train=y_train, + X_test=X_test, y_test=y_test, + optimize_metric='accuracy', + total_walltime_limit=150, + func_eval_time_limit_secs=50, + enable_traditional_pipeline=True, + load_models=False, + ) config, results = estimator.get_incumbent_results(include_traditional=include_traditional) assert isinstance(config, Configuration) assert isinstance(results, dict) diff --git a/test/test_api/utils.py b/test/test_api/utils.py index 0fa84b55b..d68f26dfa 100644 --- a/test/test_api/utils.py +++ b/test/test_api/utils.py @@ -1,3 +1,7 @@ +import os + +from smac.runhistory.runhistory import RunHistory, DataOrigin + from autoPyTorch.constants import REGRESSION_TASKS from autoPyTorch.evaluation.abstract_evaluator import ( DummyClassificationPipeline, @@ -5,6 +9,19 @@ fit_and_suppress_warnings ) from autoPyTorch.evaluation.train_evaluator import TrainEvaluator +from autoPyTorch.pipeline.traditional_tabular_classification import TraditionalTabularClassificationPipeline + + +def dummy_traditional_classification(self, time_left: int, func_eval_time_limit_secs: int) -> None: + run_history = RunHistory() + run_history.load_json('./.tmp_api/traditional_run_history.json', + TraditionalTabularClassificationPipeline(dataset_properties={ + 'numerical_columns': [10] + }).get_hyperparameter_search_space()) + self.run_history.update(run_history, DataOrigin.EXTERNAL_SAME_INSTANCES) + run_history.save_json(os.path.join(self._backend.internals_directory, 'traditional_run_history.json'), + save_external=True) + return # ======== From 7bbddede0fc875b76c732186c6849ef09a6c5110 Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Thu, 29 Apr 2021 15:03:29 +0200 Subject: [PATCH 07/16] Fixed path of run history test_api --- test/test_api/test_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_api/test_api.py b/test/test_api/test_api.py index 13d6eaaf2..2d56004b7 100644 --- a/test/test_api/test_api.py +++ b/test/test_api/test_api.py @@ -608,7 +608,7 @@ def test_get_incumbent_results(dataset_name, backend, include_traditional): ) pipeline_run_history = RunHistory() - pipeline_run_history.load_json('./.tmp_api/runhistory.json', estimator.get_search_space(dataset)) + pipeline_run_history.load_json('test/test_api/.tmp_api/runhistory.json', estimator.get_search_space(dataset)) estimator._do_dummy_prediction = unittest.mock.MagicMock() From 564628372f9b89a55f3abc0d8cc49c8f8761ef86 Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Fri, 7 May 2021 11:07:33 +0200 Subject: [PATCH 08/16] Addressed comments from Fransisco, making better tests --- autoPyTorch/api/base_task.py | 5 +- autoPyTorch/evaluation/abstract_evaluator.py | 7 +- autoPyTorch/evaluation/train_evaluator.py | 6 +- .../.tmp_api/traditional_run_history.json | 221 ++++++++++++++++++ test/test_api/test_api.py | 8 +- 5 files changed, 236 insertions(+), 11 deletions(-) create mode 100644 test/test_api/.tmp_api/traditional_run_history.json diff --git a/autoPyTorch/api/base_task.py b/autoPyTorch/api/base_task.py index cd817bf75..3160cdcd9 100644 --- a/autoPyTorch/api/base_task.py +++ b/autoPyTorch/api/base_task.py @@ -640,7 +640,6 @@ def _do_traditional_prediction(self, time_left: int, func_eval_time_limit_secs: configuration = additional_info['pipeline_configuration'] origin = additional_info['configuration_origin'] additional_info.pop('pipeline_configuration') - additional_info.pop('configuration_origin') run_history.add(config=configuration, cost=cost, time=runtime, status=status, seed=self.seed, starttime=starttime, endtime=starttime + runtime, @@ -1281,8 +1280,8 @@ def get_incumbent_results( if not include_traditional: # traditional classifiers have trainer_configuration in their additional info run_history_data = dict( - filter(lambda elem: elem[1].additional_info is not None and 'trainer_configuration' not in elem[1]. - additional_info, + filter(lambda elem: elem[1].additional_info is not None and elem[1]. + additional_info['configuration_origin'] != 'traditional', run_history_data.items())) sorted_runvalue_by_cost = sorted(run_history_data.items(), key=lambda item: item[1].cost) incumbent_run_key, incumbent_run_value = sorted_runvalue_by_cost[0] diff --git a/autoPyTorch/evaluation/abstract_evaluator.py b/autoPyTorch/evaluation/abstract_evaluator.py index d98c18fc9..b5ad43206 100644 --- a/autoPyTorch/evaluation/abstract_evaluator.py +++ b/autoPyTorch/evaluation/abstract_evaluator.py @@ -114,7 +114,8 @@ def get_additional_run_info(self) -> Dict[str, Any]: # pylint: disable=R0201 Can be found in autoPyTorch/pipeline/components/setup/traditional_ml/classifier_configs """ return {'pipeline_configuration': self.configuration, - 'trainer_configuration': self.pipeline.named_steps['model_trainer'].choice.model.get_config()} + 'trainer_configuration': self.pipeline.named_steps['model_trainer'].choice.model.get_config(), + 'configuration_origin': 'traditional'} def get_pipeline_representation(self) -> Dict[str, str]: return self.pipeline.get_pipeline_representation() @@ -178,7 +179,7 @@ def estimator_supports_iterative_fit(self) -> bool: # pylint: disable=R0201 return False def get_additional_run_info(self) -> Dict: # pylint: disable=R0201 - return {} + return {'configuration_origin': 'DUMMY'} def get_pipeline_representation(self) -> Dict[str, str]: return { @@ -237,7 +238,7 @@ def estimator_supports_iterative_fit(self) -> bool: # pylint: disable=R0201 return False def get_additional_run_info(self) -> Dict: # pylint: disable=R0201 - return {} + return {'configuration_origin': 'DUMMY'} @staticmethod def get_default_pipeline_options() -> Dict[str, Any]: diff --git a/autoPyTorch/evaluation/train_evaluator.py b/autoPyTorch/evaluation/train_evaluator.py index 4c026a155..2c5459beb 100644 --- a/autoPyTorch/evaluation/train_evaluator.py +++ b/autoPyTorch/evaluation/train_evaluator.py @@ -172,10 +172,8 @@ def fit_predict_and_loss(self) -> None: status = StatusType.SUCCESS - self.logger.debug("In train evaluator fit_predict_and_loss, num_run: {} loss:{}".format( - self.num_run, - loss - )) + self.logger.debug("In train evaluator fit_predict_and_loss, num_run: {} loss:{}," + " additional run info:{}".format(self.num_run, loss, additional_run_info)) self.finish_up( loss=loss, train_loss=train_loss, diff --git a/test/test_api/.tmp_api/traditional_run_history.json b/test/test_api/.tmp_api/traditional_run_history.json new file mode 100644 index 000000000..f7a106c82 --- /dev/null +++ b/test/test_api/.tmp_api/traditional_run_history.json @@ -0,0 +1,221 @@ +{ + "data": [ + [ + [ + 1, + null, + 1, + 0.0 + ], + [ + 0.20467836257309946, + 48.634921073913574, + { + "__enum__": "StatusType.SUCCESS" + }, + 0.0, + 0.0, + { + "trainer_configuration": { + "num_rounds": 10000, + "num_leaves": 128, + "two_round": true, + "min_data_in_leaf": 3, + "feature_fraction": 0.9, + "boosting_type": "gbdt", + "learning_rate": 0.03, + "num_threads": -1 + }, + "configuration_origin": "traditional", + "opt_loss": { + "accuracy": 0.20467836257309946, + "balanced_accuracy": 0.20607553366174058, + "roc_auc": 0.0847016967706623, + "average_precision": 0.0827746781030202, + "log_loss": 0.4155085084208271, + "precision": 0.15492957746478875, + "precision_macro": 0.19746478873239437, + "precision_micro": 0.20467836257309946, + "precision_weighted": 0.19821102050901895, + "recall": 0.2857142857142857, + "recall_macro": 0.20607553366174058, + "recall_micro": 0.20467836257309946, + "recall_weighted": 0.20467836257309946, + "f1": 0.22580645161290325, + "f1_macro": 0.2064861135069863, + "f1_micro": 0.20467836257309946, + "f1_weighted": 0.2061471602068825 + }, + "duration": 30.787471771240234, + "num_run": 2, + "train_loss": { + "accuracy": 0.0, + "balanced_accuracy": 0.0, + "roc_auc": 0.0, + "average_precision": 0.0, + "log_loss": 0.1163032455208329, + "precision": 0.0, + "precision_macro": 0.0, + "precision_micro": 0.0, + "precision_weighted": 0.0, + "recall": 0.0, + "recall_macro": 0.0, + "recall_micro": 0.0, + "recall_weighted": 0.0, + "f1": 0.0, + "f1_macro": 0.0, + "f1_micro": 0.0, + "f1_weighted": 0.0 + }, + "test_loss": 0.138728323699422 + } + ] + ], + [ + [ + 2, + null, + 1, + 0.0 + ], + [ + 0.14619883040935677, + 24.41903591156006, + { + "__enum__": "StatusType.SUCCESS" + }, + 0.0, + 0.0, + { + "trainer_configuration": { + "iterations": 10000, + "learning_rate": 0.1, + "eval_metric": "Accuracy" + }, + "configuration_origin": "traditional", + "opt_loss": { + "accuracy": 0.14619883040935677, + "balanced_accuracy": 0.14573070607553373, + "roc_auc": 0.09530651340996166, + "average_precision": 0.09777406254428278, + "log_loss": 0.5589205214851781, + "precision": 0.1685393258426966, + "precision_macro": 0.1452452726774458, + "precision_micro": 0.14619883040935677, + "precision_weighted": 0.1448366050780555, + "recall": 0.11904761904761907, + "recall_macro": 0.14573070607553373, + "recall_micro": 0.14619883040935677, + "recall_weighted": 0.14619883040935677, + "f1": 0.1445086705202312, + "f1_macro": 0.14621883230153576, + "f1_micro": 0.14619883040935677, + "f1_weighted": 0.14624883513980425 + }, + "duration": 9.664803266525269, + "num_run": 3, + "train_loss": { + "accuracy": 0.138728323699422, + "balanced_accuracy": 0.13000374748748, + "roc_auc": 0.05154498688379383, + "average_precision": 0.05783407475676716, + "log_loss": 0.5370512441920408, + "precision": 0.21468926553672318, + "precision_macro": 0.13693043158492957, + "precision_micro": 0.138728323699422, + "precision_weighted": 0.1261430788979756, + "recall": 0.06711409395973156, + "recall_macro": 0.13000374748748, + "recall_micro": 0.138728323699422, + "recall_weighted": 0.138728323699422, + "f1": 0.1472392638036809, + "f1_macro": 0.13919340239364375, + "f1_micro": 0.138728323699422, + "f1_weighted": 0.13807721352751146 + }, + "test_loss": 0.12716763005780352 + } + ] + ], + [ + [ + 3, + null, + 1, + 0.0 + ], + [ + 0.14035087719298245, + 18.845818758010864, + { + "__enum__": "StatusType.SUCCESS" + }, + 0.0, + 0.0, + { + "trainer_configuration": { + "n_estimators": 300, + "n_jobs": -1 + }, + "configuration_origin": "traditional", + "opt_loss": { + "accuracy": 0.14035087719298245, + "balanced_accuracy": 0.14142036124794743, + "roc_auc": 0.08401751505199773, + "average_precision": 0.0788213312884698, + "log_loss": 0.37833770927673543, + "precision": 0.09459459459459463, + "precision_macro": 0.13492616327667872, + "precision_micro": 0.14035087719298245, + "precision_weighted": 0.1356337346570663, + "recall": 0.20238095238095233, + "recall_macro": 0.14142036124794743, + "recall_micro": 0.14035087719298245, + "recall_weighted": 0.14035087719298245, + "f1": 0.15189873417721522, + "f1_macro": 0.14116675839295545, + "f1_micro": 0.14035087719298245, + "f1_weighted": 0.1409784781160387 + }, + "duration": 4.936332941055298, + "num_run": 4, + "train_loss": { + "accuracy": 0.0, + "balanced_accuracy": 0.0, + "roc_auc": 0.0, + "average_precision": 2.220446049250313e-16, + "log_loss": 0.0899028721860357, + "precision": 0.0, + "precision_macro": 0.0, + "precision_micro": 0.0, + "precision_weighted": 0.0, + "recall": 0.0, + "recall_macro": 0.0, + "recall_micro": 0.0, + "recall_weighted": 0.0, + "f1": 0.0, + "f1_macro": 0.0, + "f1_micro": 0.0, + "f1_weighted": 0.0 + }, + "test_loss": 0.1445086705202312 + } + ] + ] + ], + "config_origins": {}, + "configs": { + "1": { + "model_trainer:__choice__": "tabular_classifier", + "model_trainer:tabular_classifier:classifier": "lgb" + }, + "2": { + "model_trainer:__choice__": "tabular_classifier", + "model_trainer:tabular_classifier:classifier": "catboost" + }, + "3": { + "model_trainer:__choice__": "tabular_classifier", + "model_trainer:tabular_classifier:classifier": "random_forest" + } + } +} \ No newline at end of file diff --git a/test/test_api/test_api.py b/test/test_api/test_api.py index 2d56004b7..d2a131c24 100644 --- a/test/test_api/test_api.py +++ b/test/test_api/test_api.py @@ -608,7 +608,8 @@ def test_get_incumbent_results(dataset_name, backend, include_traditional): ) pipeline_run_history = RunHistory() - pipeline_run_history.load_json('test/test_api/.tmp_api/runhistory.json', estimator.get_search_space(dataset)) + pipeline_run_history.load_json(os.path.join(os.path.dirname(__file__), '.tmp_api/runhistory.json'), + estimator.get_search_space(dataset)) estimator._do_dummy_prediction = unittest.mock.MagicMock() @@ -629,5 +630,10 @@ def test_get_incumbent_results(dataset_name, backend, include_traditional): assert isinstance(config, Configuration) assert isinstance(results, dict) + run_history_data = estimator.run_history.data + costs = [run_value.cost for run_key, run_value in run_history_data.items() if run_value.additional_info is not None + and (run_value.additional_info['configuration_origin'] != 'traditional' or include_traditional)] + assert results['opt_loss']['accuracy'] == min(costs) + if not include_traditional: assert results['configuration_origin'] != 'traditional' From 6e1bbca1ba96e2481fe95989d504cf8bae29c93b Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Fri, 7 May 2021 11:13:32 +0200 Subject: [PATCH 09/16] fix flake --- test/test_api/test_api.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/test_api/test_api.py b/test/test_api/test_api.py index d2a131c24..3954be31a 100644 --- a/test/test_api/test_api.py +++ b/test/test_api/test_api.py @@ -8,6 +8,8 @@ from ConfigSpace.configuration_space import Configuration +from api_utils import dummy_traditional_classification + import numpy as np import pandas as pd From fab0608ac4355f596b714284c690c26c37b072ed Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Mon, 17 May 2021 23:32:39 +0200 Subject: [PATCH 10/16] After rebase fix issues --- test/test_api/.tmp_api/runhistory.json | 1080 ++++++++++++++++++++++++ test/test_api/test_api.py | 2 - 2 files changed, 1080 insertions(+), 2 deletions(-) create mode 100644 test/test_api/.tmp_api/runhistory.json diff --git a/test/test_api/.tmp_api/runhistory.json b/test/test_api/.tmp_api/runhistory.json new file mode 100644 index 000000000..6f61e1395 --- /dev/null +++ b/test/test_api/.tmp_api/runhistory.json @@ -0,0 +1,1080 @@ +{ + "data": [ + [ + [ + 1, + "{\"task_id\": \"954f8906efe419f4acae1bb38a8dd00a\"}", + 0, + 5.555555555555555 + ], + [ + 0.33999999999999997, + 6.753758192062378, + { + "__enum__": "StatusType.SUCCESS" + }, + 1619605596.255282, + 1619605604.6635818, + { + "opt_loss": { + "accuracy": 0.33999999999999997, + "balanced_accuracy": 0.3148148148148149, + "roc_auc": 0.2753623188405797, + "average_precision": 0.30732687186723573, + "log_loss": 0.6089426457881928, + "precision": 0.42500000000000004, + "precision_macro": 0.21250000000000002, + "precision_micro": 0.33999999999999997, + "precision_weighted": 0.1955, + "recall": 0.0, + "recall_macro": 0.3148148148148149, + "recall_micro": 0.33999999999999997, + "recall_weighted": 0.33999999999999997, + "f1": 0.2698412698412699, + "f1_macro": 0.3646503646503647, + "f1_micro": 0.33999999999999997, + "f1_weighted": 0.3722350922350922 + }, + "duration": 1.0248379707336426, + "num_run": 7, + "train_loss": { + "accuracy": 0.18999999999999995, + "balanced_accuracy": 0.19791666666666674, + "roc_auc": 0.13501602564102555, + "average_precision": 0.12605087038765284, + "log_loss": 0.5467884422838688, + "precision": 0.2676056338028169, + "precision_macro": 0.1338028169014085, + "precision_micro": 0.18999999999999995, + "precision_weighted": 0.13915492957746478, + "recall": 0.0, + "recall_macro": 0.19791666666666674, + "recall_micro": 0.18999999999999995, + "recall_weighted": 0.18999999999999995, + "f1": 0.1544715447154471, + "f1_macro": 0.20061239573434686, + "f1_micro": 0.18999999999999995, + "f1_weighted": 0.19876676169359087 + }, + "test_loss": 0.26, + "configuration_origin": "Default" + } + ] + ], + [ + [ + 2, + "{\"task_id\": \"954f8906efe419f4acae1bb38a8dd00a\"}", + 0, + 5.555555555555555 + ], + [ + 0.54, + 13.387118101119995, + { + "__enum__": "StatusType.SUCCESS" + }, + 1619605604.8070002, + 1619605619.942064, + { + "opt_loss": { + "accuracy": 0.54, + "balanced_accuracy": 0.5, + "roc_auc": 0.5, + "average_precision": 0.54, + "log_loss": 9.143907852172852, + "precision": 0.54, + "precision_macro": 0.77, + "precision_micro": 0.54, + "precision_weighted": 0.7884, + "recall": 0.0, + "recall_macro": 0.5, + "recall_micro": 0.54, + "recall_weighted": 0.54, + "f1": 0.36986301369863006, + "f1_macro": 0.684931506849315, + "f1_micro": 0.54, + "f1_weighted": 0.7101369863013698 + }, + "duration": 6.89619779586792, + "num_run": 8, + "train_loss": { + "accuracy": 0.48, + "balanced_accuracy": 0.5, + "roc_auc": 0.5, + "average_precision": 0.48, + "log_loss": 8.127918090820312, + "precision": 0.48, + "precision_macro": 0.74, + "precision_micro": 0.48, + "precision_weighted": 0.7296, + "recall": 0.0, + "recall_macro": 0.5, + "recall_micro": 0.48, + "recall_weighted": 0.48, + "f1": 0.3157894736842105, + "f1_macro": 0.6578947368421053, + "f1_micro": 0.48, + "f1_weighted": 0.6442105263157895 + }, + "test_loss": 0.48, + "configuration_origin": "Random Search" + } + ] + ], + [ + [ + 3, + "{\"task_id\": \"954f8906efe419f4acae1bb38a8dd00a\"}", + 0, + 5.555555555555555 + ], + [ + 0.45999999999999996, + 6.888725996017456, + { + "__enum__": "StatusType.SUCCESS" + }, + 1619605624.0454779, + 1619605632.6947682, + { + "opt_loss": { + "accuracy": 0.45999999999999996, + "balanced_accuracy": 0.5, + "roc_auc": 0.39291465378421897, + "average_precision": 0.44602082475468785, + "log_loss": 4.61612025549206, + "precision": 1.0, + "precision_macro": 0.73, + "precision_micro": 0.45999999999999996, + "precision_weighted": 0.7083999999999999, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.45999999999999996, + "recall_weighted": 0.45999999999999996, + "f1": 1.0, + "f1_macro": 0.6493506493506493, + "f1_micro": 0.45999999999999996, + "f1_weighted": 0.6212987012987012 + }, + "duration": 1.0771801471710205, + "num_run": 9, + "train_loss": { + "accuracy": 0.52, + "balanced_accuracy": 0.5, + "roc_auc": 0.29006410256410264, + "average_precision": 0.2868909098817277, + "log_loss": 5.2043224499779175, + "precision": 1.0, + "precision_macro": 0.76, + "precision_micro": 0.52, + "precision_weighted": 0.7696000000000001, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.52, + "recall_weighted": 0.52, + "f1": 1.0, + "f1_macro": 0.6756756756756757, + "f1_micro": 0.52, + "f1_weighted": 0.6886486486486486 + }, + "test_loss": 0.52, + "configuration_origin": "Random Search (sorted)" + } + ] + ], + [ + [ + 4, + "{\"task_id\": \"954f8906efe419f4acae1bb38a8dd00a\"}", + 0, + 5.555555555555555 + ], + [ + 0.19999999999999996, + 7.270266056060791, + { + "__enum__": "StatusType.SUCCESS" + }, + 1619605632.739142, + 1619605642.037219, + { + "opt_loss": { + "accuracy": 0.19999999999999996, + "balanced_accuracy": 0.18840579710144922, + "roc_auc": 0.1561996779388083, + "average_precision": 0.21604680412442745, + "log_loss": 0.5398924820125103, + "precision": 0.29032258064516125, + "precision_macro": 0.1714770797962648, + "precision_micro": 0.19999999999999996, + "precision_weighted": 0.1619694397283531, + "recall": 0.04347826086956519, + "recall_macro": 0.18840579710144922, + "recall_micro": 0.19999999999999996, + "recall_weighted": 0.19999999999999996, + "f1": 0.18518518518518512, + "f1_macro": 0.20128824476650564, + "f1_micro": 0.19999999999999984, + "f1_weighted": 0.20257648953301133 + }, + "duration": 1.4017798900604248, + "num_run": 10, + "train_loss": { + "accuracy": 0.13, + "balanced_accuracy": 0.12900641025641024, + "roc_auc": 0.07772435897435892, + "average_precision": 0.08652341359886084, + "log_loss": 0.5315199671685695, + "precision": 0.10204081632653061, + "precision_macro": 0.12945178071228491, + "precision_micro": 0.13, + "precision_weighted": 0.12835534213685473, + "recall": 0.15384615384615385, + "recall_macro": 0.12900641025641024, + "recall_micro": 0.13, + "recall_weighted": 0.13, + "f1": 0.12871287128712872, + "f1_macro": 0.13001300130013005, + "f1_micro": 0.13, + "f1_weighted": 0.12996099609960998 + }, + "test_loss": 0.18000000000000005, + "configuration_origin": "Random Search (sorted)" + } + ] + ], + [ + [ + 5, + "{\"task_id\": \"954f8906efe419f4acae1bb38a8dd00a\"}", + 0, + 5.555555555555555 + ], + [ + 0.54, + 7.0798258781433105, + { + "__enum__": "StatusType.SUCCESS" + }, + 1619605645.7424152, + 1619605654.4772348, + { + "opt_loss": { + "accuracy": 0.54, + "balanced_accuracy": 0.5, + "roc_auc": 0.48309178743961356, + "average_precision": 0.5165588473668532, + "log_loss": 0.8714349645376206, + "precision": 0.54, + "precision_macro": 0.77, + "precision_micro": 0.54, + "precision_weighted": 0.7884, + "recall": 0.0, + "recall_macro": 0.5, + "recall_micro": 0.54, + "recall_weighted": 0.54, + "f1": 0.36986301369863006, + "f1_macro": 0.684931506849315, + "f1_micro": 0.54, + "f1_weighted": 0.7101369863013698 + }, + "duration": 1.3237178325653076, + "num_run": 11, + "train_loss": { + "accuracy": 0.48, + "balanced_accuracy": 0.5, + "roc_auc": 0.45552884615384626, + "average_precision": 0.3374851931072518, + "log_loss": 0.8075011664628983, + "precision": 0.48, + "precision_macro": 0.74, + "precision_micro": 0.48, + "precision_weighted": 0.7296, + "recall": 0.0, + "recall_macro": 0.5, + "recall_micro": 0.48, + "recall_weighted": 0.48, + "f1": 0.3157894736842105, + "f1_macro": 0.6578947368421053, + "f1_micro": 0.48, + "f1_weighted": 0.6442105263157895 + }, + "test_loss": 0.48, + "configuration_origin": "Random Search" + } + ] + ], + [ + [ + 6, + "{\"task_id\": \"954f8906efe419f4acae1bb38a8dd00a\"}", + 0, + 5.555555555555555 + ], + [ + 0.45999999999999996, + 7.803380966186523, + { + "__enum__": "StatusType.SUCCESS" + }, + 1619605654.525853, + 1619605664.386794, + { + "opt_loss": { + "accuracy": 0.45999999999999996, + "balanced_accuracy": 0.5, + "roc_auc": 0.49436392914653793, + "average_precision": 0.5017321175078623, + "log_loss": 0.8439558809995651, + "precision": 1.0, + "precision_macro": 0.73, + "precision_micro": 0.45999999999999996, + "precision_weighted": 0.7083999999999999, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.45999999999999996, + "recall_weighted": 0.45999999999999996, + "f1": 1.0, + "f1_macro": 0.6493506493506493, + "f1_micro": 0.45999999999999996, + "f1_weighted": 0.6212987012987012 + }, + "duration": 2.0095207691192627, + "num_run": 12, + "train_loss": { + "accuracy": 0.52, + "balanced_accuracy": 0.5, + "roc_auc": 0.5841346153846154, + "average_precision": 0.5124520321953292, + "log_loss": 0.9231552585959435, + "precision": 1.0, + "precision_macro": 0.76, + "precision_micro": 0.52, + "precision_weighted": 0.7696000000000001, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.52, + "recall_weighted": 0.52, + "f1": 1.0, + "f1_macro": 0.6756756756756757, + "f1_micro": 0.52, + "f1_weighted": 0.6886486486486486 + }, + "test_loss": 0.52, + "configuration_origin": "Random Search" + } + ] + ], + [ + [ + 7, + "{\"task_id\": \"954f8906efe419f4acae1bb38a8dd00a\"}", + 0, + 5.555555555555555 + ], + [ + 0.36, + 7.244281053543091, + { + "__enum__": "StatusType.SUCCESS" + }, + 1619605668.36801, + 1619605677.596504, + { + "opt_loss": { + "accuracy": 0.36, + "balanced_accuracy": 0.3429951690821256, + "roc_auc": 0.3317230273752013, + "average_precision": 0.3484537959109295, + "log_loss": 0.6334248012304307, + "precision": 0.4285714285714286, + "precision_macro": 0.3142857142857143, + "precision_micro": 0.36, + "precision_weighted": 0.30514285714285705, + "recall": 0.13043478260869568, + "recall_macro": 0.3429951690821256, + "recall_micro": 0.36, + "recall_weighted": 0.36, + "f1": 0.31034482758620696, + "f1_macro": 0.36945812807881784, + "f1_micro": 0.36, + "f1_weighted": 0.3741871921182266 + }, + "duration": 1.5409259796142578, + "num_run": 13, + "train_loss": { + "accuracy": 0.22999999999999998, + "balanced_accuracy": 0.23637820512820507, + "roc_auc": 0.16947115384615385, + "average_precision": 0.14773539974411132, + "log_loss": 0.5595843681693077, + "precision": 0.28358208955223885, + "precision_macro": 0.20239710538218003, + "precision_micro": 0.22999999999999998, + "precision_weighted": 0.20564450474898233, + "recall": 0.07692307692307687, + "recall_macro": 0.23637820512820507, + "recall_micro": 0.22999999999999998, + "recall_weighted": 0.22999999999999998, + "f1": 0.19327731092436973, + "f1_macro": 0.23861396410416025, + "f1_micro": 0.2300000000000001, + "f1_weighted": 0.23680049797696856 + }, + "test_loss": 0.33999999999999997, + "configuration_origin": "Random Search" + } + ] + ], + [ + [ + 8, + "{\"task_id\": \"954f8906efe419f4acae1bb38a8dd00a\"}", + 0, + 5.555555555555555 + ], + [ + 0.45999999999999996, + 16.007498025894165, + { + "__enum__": "StatusType.SUCCESS" + }, + 1619605677.646837, + 1619605695.31683, + { + "opt_loss": { + "accuracy": 0.45999999999999996, + "balanced_accuracy": 0.5, + "roc_auc": 0.5, + "average_precision": 0.54, + "log_loss": 9.349400329589844, + "precision": 1.0, + "precision_macro": 0.73, + "precision_micro": 0.45999999999999996, + "precision_weighted": 0.7083999999999999, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.45999999999999996, + "recall_weighted": 0.45999999999999996, + "f1": 1.0, + "f1_macro": 0.6493506493506493, + "f1_micro": 0.45999999999999996, + "f1_weighted": 0.6212987012987012 + }, + "duration": 9.252167224884033, + "num_run": 14, + "train_loss": { + "accuracy": 0.52, + "balanced_accuracy": 0.5, + "roc_auc": 0.5, + "average_precision": 0.48, + "log_loss": 10.568887329101562, + "precision": 1.0, + "precision_macro": 0.76, + "precision_micro": 0.52, + "precision_weighted": 0.7696000000000001, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.52, + "recall_weighted": 0.52, + "f1": 1.0, + "f1_macro": 0.6756756756756757, + "f1_micro": 0.52, + "f1_weighted": 0.6886486486486486 + }, + "test_loss": 0.52, + "configuration_origin": "Random Search" + } + ] + ], + [ + [ + 9, + "{\"task_id\": \"954f8906efe419f4acae1bb38a8dd00a\"}", + 0, + 5.555555555555555 + ], + [ + 0.54, + 10.220438718795776, + { + "__enum__": "StatusType.SUCCESS" + }, + 1619605701.1320379, + 1619605712.934675, + { + "opt_loss": { + "accuracy": 0.54, + "balanced_accuracy": 0.5, + "roc_auc": 0.32367149758454117, + "average_precision": 0.34546042899995555, + "log_loss": 0.9921851700544357, + "precision": 0.54, + "precision_macro": 0.77, + "precision_micro": 0.54, + "precision_weighted": 0.7884, + "recall": 0.0, + "recall_macro": 0.5, + "recall_micro": 0.54, + "recall_weighted": 0.54, + "f1": 0.36986301369863006, + "f1_macro": 0.684931506849315, + "f1_micro": 0.54, + "f1_weighted": 0.7101369863013698 + }, + "duration": 3.410412073135376, + "num_run": 15, + "train_loss": { + "accuracy": 0.48, + "balanced_accuracy": 0.5, + "roc_auc": 0.2736378205128205, + "average_precision": 0.3503510059229116, + "log_loss": 0.8750112408399582, + "precision": 0.48, + "precision_macro": 0.74, + "precision_micro": 0.48, + "precision_weighted": 0.7296, + "recall": 0.0, + "recall_macro": 0.5, + "recall_micro": 0.48, + "recall_weighted": 0.48, + "f1": 0.3157894736842105, + "f1_macro": 0.6578947368421053, + "f1_micro": 0.48, + "f1_weighted": 0.6442105263157895 + }, + "test_loss": 0.48, + "configuration_origin": "Random Search (sorted)" + } + ] + ], + [ + [ + 4, + "{\"task_id\": \"954f8906efe419f4acae1bb38a8dd00a\"}", + 0, + 16.666666666666664 + ], + [ + 0.19999999999999996, + 7.500712156295776, + { + "__enum__": "StatusType.SUCCESS" + }, + 1619605716.9696152, + 1619605726.203081, + { + "opt_loss": { + "accuracy": 0.19999999999999996, + "balanced_accuracy": 0.18518518518518512, + "roc_auc": 0.1497584541062802, + "average_precision": 0.20785742506548321, + "log_loss": 0.47934962511062623, + "precision": 0.303030303030303, + "precision_macro": 0.1515151515151515, + "precision_micro": 0.19999999999999996, + "precision_weighted": 0.1393939393939394, + "recall": 0.0, + "recall_macro": 0.18518518518518512, + "recall_micro": 0.19999999999999996, + "recall_weighted": 0.19999999999999996, + "f1": 0.1785714285714285, + "f1_macro": 0.20292207792207795, + "f1_micro": 0.19999999999999984, + "f1_weighted": 0.20487012987012976 + }, + "duration": 1.825955867767334, + "num_run": 10, + "train_loss": { + "accuracy": 0.14, + "balanced_accuracy": 0.14423076923076916, + "roc_auc": 0.07692307692307698, + "average_precision": 0.08536482019177061, + "log_loss": 0.3631942201941274, + "precision": 0.19354838709677424, + "precision_macro": 0.1230899830220713, + "precision_micro": 0.14, + "precision_weighted": 0.12590831918505951, + "recall": 0.038461538461538436, + "recall_macro": 0.14423076923076916, + "recall_micro": 0.14, + "recall_weighted": 0.14, + "f1": 0.1228070175438597, + "f1_macro": 0.14279885760913913, + "f1_micro": 0.14000000000000012, + "f1_weighted": 0.1419991840065279 + }, + "test_loss": 0.24, + "configuration_origin": "Random Search (sorted)" + } + ] + ], + [ + [ + 1, + "{\"task_id\": \"954f8906efe419f4acae1bb38a8dd00a\"}", + 0, + 16.666666666666664 + ], + [ + 0.18000000000000005, + 6.8435540199279785, + { + "__enum__": "StatusType.SUCCESS" + }, + 1619605726.252876, + 1619605734.6368818, + { + "opt_loss": { + "accuracy": 0.18000000000000005, + "balanced_accuracy": 0.17632850241545894, + "roc_auc": 0.06441223832528187, + "average_precision": 0.07524038362673102, + "log_loss": 0.6117494863834065, + "precision": 0.23076923076923073, + "precision_macro": 0.17788461538461542, + "precision_micro": 0.18000000000000005, + "precision_weighted": 0.17365384615384616, + "recall": 0.13043478260869568, + "recall_macro": 0.17632850241545894, + "recall_micro": 0.18000000000000005, + "recall_weighted": 0.18000000000000005, + "f1": 0.18367346938775508, + "f1_macro": 0.1800720288115245, + "f1_micro": 0.18000000000000005, + "f1_weighted": 0.179783913565426 + }, + "duration": 1.1207940578460693, + "num_run": 7, + "train_loss": { + "accuracy": 0.10999999999999999, + "balanced_accuracy": 0.10817307692307687, + "roc_auc": 0.034455128205128194, + "average_precision": 0.0423390464532285, + "log_loss": 0.4021380259158622, + "precision": 0.06382978723404253, + "precision_macro": 0.1073865917302288, + "precision_micro": 0.10999999999999999, + "precision_weighted": 0.10564431955038134, + "recall": 0.15384615384615385, + "recall_macro": 0.10817307692307687, + "recall_micro": 0.10999999999999999, + "recall_weighted": 0.10999999999999999, + "f1": 0.11111111111111116, + "f1_macro": 0.11001100110011008, + "f1_micro": 0.10999999999999999, + "f1_weighted": 0.11005500550055003 + }, + "test_loss": 0.06000000000000005, + "configuration_origin": "Default" + } + ] + ], + [ + [ + 7, + "{\"task_id\": \"954f8906efe419f4acae1bb38a8dd00a\"}", + 0, + 16.666666666666664 + ], + [ + 2147483647.0, + 0.0, + { + "__enum__": "StatusType.RUNNING" + }, + 0.0, + 0.0, + null + ] + ] + ], + "config_origins": { + "1": "Default", + "2": "Random Search", + "3": "Random Search (sorted)", + "4": "Random Search (sorted)", + "5": "Random Search", + "6": "Random Search", + "7": "Random Search", + "8": "Random Search", + "9": "Random Search (sorted)" + }, + "configs": { + "1": { + "data_loader:batch_size": 64, + "encoder:__choice__": "NoEncoder", + "feature_preprocessor:__choice__": "NoFeaturePreprocessor", + "imputer:numerical_strategy": "mean", + "lr_scheduler:__choice__": "ReduceLROnPlateau", + "network_backbone:__choice__": "ShapedMLPBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "XavierInit", + "optimizer:__choice__": "AdamOptimizer", + "scaler:__choice__": "StandardScaler", + "trainer:__choice__": "StandardTrainer", + "lr_scheduler:ReduceLROnPlateau:factor": 0.1, + "lr_scheduler:ReduceLROnPlateau:mode": "min", + "lr_scheduler:ReduceLROnPlateau:patience": 10, + "network_backbone:ShapedMLPBackbone:activation": "relu", + "network_backbone:ShapedMLPBackbone:max_units": 200, + "network_backbone:ShapedMLPBackbone:mlp_shape": "funnel", + "network_backbone:ShapedMLPBackbone:num_groups": 5, + "network_backbone:ShapedMLPBackbone:output_dim": 200, + "network_backbone:ShapedMLPBackbone:use_dropout": false, + "network_head:fully_connected:num_layers": 2, + "network_init:XavierInit:bias_strategy": "Normal", + "optimizer:AdamOptimizer:beta1": 0.9, + "optimizer:AdamOptimizer:beta2": 0.9, + "optimizer:AdamOptimizer:lr": 0.01, + "optimizer:AdamOptimizer:weight_decay": 0.0, + "trainer:StandardTrainer:weighted_loss": true, + "network_head:fully_connected:activation": "relu", + "network_head:fully_connected:units_layer_1": 128 + }, + "2": { + "data_loader:batch_size": 101, + "encoder:__choice__": "NoEncoder", + "feature_preprocessor:__choice__": "PowerTransformer", + "imputer:numerical_strategy": "most_frequent", + "lr_scheduler:__choice__": "CyclicLR", + "network_backbone:__choice__": "ResNetBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "XavierInit", + "optimizer:__choice__": "AdamWOptimizer", + "scaler:__choice__": "StandardScaler", + "trainer:__choice__": "StandardTrainer", + "feature_preprocessor:PowerTransformer:standardize": true, + "lr_scheduler:CyclicLR:base_lr": 0.04449008283592498, + "lr_scheduler:CyclicLR:max_lr": 0.04072677124783613, + "lr_scheduler:CyclicLR:mode": "triangular2", + "lr_scheduler:CyclicLR:step_size_up": 1628, + "network_backbone:ResNetBackbone:activation": "relu", + "network_backbone:ResNetBackbone:blocks_per_group_0": 4, + "network_backbone:ResNetBackbone:blocks_per_group_1": 2, + "network_backbone:ResNetBackbone:num_groups": 14, + "network_backbone:ResNetBackbone:num_units_0": 347, + "network_backbone:ResNetBackbone:num_units_1": 490, + "network_backbone:ResNetBackbone:use_dropout": false, + "network_backbone:ResNetBackbone:use_shake_drop": false, + "network_backbone:ResNetBackbone:use_shake_shake": true, + "network_head:fully_connected:num_layers": 3, + "network_init:XavierInit:bias_strategy": "Normal", + "optimizer:AdamWOptimizer:beta1": 0.9368893300110837, + "optimizer:AdamWOptimizer:beta2": 0.9386663140612604, + "optimizer:AdamWOptimizer:lr": 3.1954346418711966e-05, + "optimizer:AdamWOptimizer:weight_decay": 0.020317400900740492, + "trainer:StandardTrainer:weighted_loss": true, + "network_backbone:ResNetBackbone:blocks_per_group_10": 3, + "network_backbone:ResNetBackbone:blocks_per_group_11": 2, + "network_backbone:ResNetBackbone:blocks_per_group_12": 4, + "network_backbone:ResNetBackbone:blocks_per_group_13": 4, + "network_backbone:ResNetBackbone:blocks_per_group_14": 1, + "network_backbone:ResNetBackbone:blocks_per_group_2": 1, + "network_backbone:ResNetBackbone:blocks_per_group_3": 3, + "network_backbone:ResNetBackbone:blocks_per_group_4": 1, + "network_backbone:ResNetBackbone:blocks_per_group_5": 4, + "network_backbone:ResNetBackbone:blocks_per_group_6": 4, + "network_backbone:ResNetBackbone:blocks_per_group_7": 1, + "network_backbone:ResNetBackbone:blocks_per_group_8": 1, + "network_backbone:ResNetBackbone:blocks_per_group_9": 2, + "network_backbone:ResNetBackbone:num_units_10": 345, + "network_backbone:ResNetBackbone:num_units_11": 514, + "network_backbone:ResNetBackbone:num_units_12": 688, + "network_backbone:ResNetBackbone:num_units_13": 441, + "network_backbone:ResNetBackbone:num_units_14": 631, + "network_backbone:ResNetBackbone:num_units_2": 854, + "network_backbone:ResNetBackbone:num_units_3": 57, + "network_backbone:ResNetBackbone:num_units_4": 96, + "network_backbone:ResNetBackbone:num_units_5": 410, + "network_backbone:ResNetBackbone:num_units_6": 453, + "network_backbone:ResNetBackbone:num_units_7": 884, + "network_backbone:ResNetBackbone:num_units_8": 108, + "network_backbone:ResNetBackbone:num_units_9": 877, + "network_head:fully_connected:activation": "tanh", + "network_head:fully_connected:units_layer_1": 321, + "network_head:fully_connected:units_layer_2": 286 + }, + "3": { + "data_loader:batch_size": 242, + "encoder:__choice__": "NoEncoder", + "feature_preprocessor:__choice__": "RandomKitchenSinks", + "imputer:numerical_strategy": "median", + "lr_scheduler:__choice__": "NoScheduler", + "network_backbone:__choice__": "ShapedMLPBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "KaimingInit", + "optimizer:__choice__": "RMSpropOptimizer", + "scaler:__choice__": "Normalizer", + "trainer:__choice__": "StandardTrainer", + "feature_preprocessor:RandomKitchenSinks:gamma": 0.13740994010268842, + "feature_preprocessor:RandomKitchenSinks:n_components": 2, + "network_backbone:ShapedMLPBackbone:activation": "sigmoid", + "network_backbone:ShapedMLPBackbone:max_units": 315, + "network_backbone:ShapedMLPBackbone:mlp_shape": "long_funnel", + "network_backbone:ShapedMLPBackbone:num_groups": 4, + "network_backbone:ShapedMLPBackbone:output_dim": 434, + "network_backbone:ShapedMLPBackbone:use_dropout": false, + "network_head:fully_connected:num_layers": 1, + "network_init:KaimingInit:bias_strategy": "Zero", + "optimizer:RMSpropOptimizer:alpha": 0.14539048589095063, + "optimizer:RMSpropOptimizer:lr": 0.004519636455032077, + "optimizer:RMSpropOptimizer:momentum": 0.4045104763524239, + "optimizer:RMSpropOptimizer:weight_decay": 0.052200962446096434, + "scaler:Normalizer:norm": "max", + "trainer:StandardTrainer:weighted_loss": false + }, + "4": { + "data_loader:batch_size": 115, + "encoder:__choice__": "NoEncoder", + "feature_preprocessor:__choice__": "Nystroem", + "imputer:numerical_strategy": "median", + "lr_scheduler:__choice__": "CosineAnnealingLR", + "network_backbone:__choice__": "ShapedMLPBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "OrthogonalInit", + "optimizer:__choice__": "SGDOptimizer", + "scaler:__choice__": "StandardScaler", + "trainer:__choice__": "StandardTrainer", + "feature_preprocessor:Nystroem:kernel": "poly", + "feature_preprocessor:Nystroem:n_components": 2, + "lr_scheduler:CosineAnnealingLR:T_max": 111, + "network_backbone:ShapedMLPBackbone:activation": "tanh", + "network_backbone:ShapedMLPBackbone:max_units": 828, + "network_backbone:ShapedMLPBackbone:mlp_shape": "diamond", + "network_backbone:ShapedMLPBackbone:num_groups": 9, + "network_backbone:ShapedMLPBackbone:output_dim": 31, + "network_backbone:ShapedMLPBackbone:use_dropout": true, + "network_head:fully_connected:num_layers": 1, + "network_init:OrthogonalInit:bias_strategy": "Zero", + "optimizer:SGDOptimizer:lr": 0.012406910679436106, + "optimizer:SGDOptimizer:momentum": 0.9573350590314021, + "optimizer:SGDOptimizer:weight_decay": 0.09520520861183296, + "trainer:StandardTrainer:weighted_loss": true, + "feature_preprocessor:Nystroem:coef0": 0.0318128711573642, + "feature_preprocessor:Nystroem:degree": 2, + "feature_preprocessor:Nystroem:gamma": 0.6547220602200696, + "network_backbone:ShapedMLPBackbone:max_dropout": 0.5525107301705023 + }, + "5": { + "data_loader:batch_size": 185, + "encoder:__choice__": "NoEncoder", + "feature_preprocessor:__choice__": "RandomKitchenSinks", + "imputer:numerical_strategy": "median", + "lr_scheduler:__choice__": "ReduceLROnPlateau", + "network_backbone:__choice__": "MLPBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "SparseInit", + "optimizer:__choice__": "AdamWOptimizer", + "scaler:__choice__": "Normalizer", + "trainer:__choice__": "StandardTrainer", + "feature_preprocessor:RandomKitchenSinks:gamma": 6.6101364296294784, + "feature_preprocessor:RandomKitchenSinks:n_components": 2, + "lr_scheduler:ReduceLROnPlateau:factor": 0.12185237487141365, + "lr_scheduler:ReduceLROnPlateau:mode": "min", + "lr_scheduler:ReduceLROnPlateau:patience": 20, + "network_backbone:MLPBackbone:activation": "sigmoid", + "network_backbone:MLPBackbone:num_groups": 5, + "network_backbone:MLPBackbone:num_units_1": 569, + "network_backbone:MLPBackbone:use_dropout": true, + "network_head:fully_connected:num_layers": 1, + "network_init:SparseInit:bias_strategy": "Normal", + "optimizer:AdamWOptimizer:beta1": 0.8989104711868687, + "optimizer:AdamWOptimizer:beta2": 0.9322314000736309, + "optimizer:AdamWOptimizer:lr": 0.0058859141412670795, + "optimizer:AdamWOptimizer:weight_decay": 0.07063518132936925, + "scaler:Normalizer:norm": "max", + "trainer:StandardTrainer:weighted_loss": true, + "network_backbone:MLPBackbone:dropout_1": 0.04606226805077265, + "network_backbone:MLPBackbone:dropout_2": 0.41372696817304083, + "network_backbone:MLPBackbone:dropout_3": 0.3134361709702569, + "network_backbone:MLPBackbone:dropout_4": 0.17919569613814074, + "network_backbone:MLPBackbone:dropout_5": 0.7704339366070313, + "network_backbone:MLPBackbone:num_units_2": 668, + "network_backbone:MLPBackbone:num_units_3": 250, + "network_backbone:MLPBackbone:num_units_4": 942, + "network_backbone:MLPBackbone:num_units_5": 925 + }, + "6": { + "data_loader:batch_size": 95, + "encoder:__choice__": "NoEncoder", + "feature_preprocessor:__choice__": "RandomKitchenSinks", + "imputer:numerical_strategy": "most_frequent", + "lr_scheduler:__choice__": "ExponentialLR", + "network_backbone:__choice__": "ShapedMLPBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "KaimingInit", + "optimizer:__choice__": "AdamOptimizer", + "scaler:__choice__": "StandardScaler", + "trainer:__choice__": "StandardTrainer", + "feature_preprocessor:RandomKitchenSinks:gamma": 0.011083412821565284, + "feature_preprocessor:RandomKitchenSinks:n_components": 4, + "lr_scheduler:ExponentialLR:gamma": 0.875262455811338, + "network_backbone:ShapedMLPBackbone:activation": "sigmoid", + "network_backbone:ShapedMLPBackbone:max_units": 969, + "network_backbone:ShapedMLPBackbone:mlp_shape": "brick", + "network_backbone:ShapedMLPBackbone:num_groups": 11, + "network_backbone:ShapedMLPBackbone:output_dim": 648, + "network_backbone:ShapedMLPBackbone:use_dropout": true, + "network_head:fully_connected:num_layers": 2, + "network_init:KaimingInit:bias_strategy": "Zero", + "optimizer:AdamOptimizer:beta1": 0.9762889500226449, + "optimizer:AdamOptimizer:beta2": 0.9820444999332838, + "optimizer:AdamOptimizer:lr": 0.00030119960921905133, + "optimizer:AdamOptimizer:weight_decay": 0.08223940799124967, + "trainer:StandardTrainer:weighted_loss": true, + "network_backbone:ShapedMLPBackbone:max_dropout": 0.6027581551489561, + "network_head:fully_connected:activation": "sigmoid", + "network_head:fully_connected:units_layer_1": 166 + }, + "7": { + "data_loader:batch_size": 119, + "encoder:__choice__": "NoEncoder", + "feature_preprocessor:__choice__": "Nystroem", + "imputer:numerical_strategy": "mean", + "lr_scheduler:__choice__": "StepLR", + "network_backbone:__choice__": "MLPBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "KaimingInit", + "optimizer:__choice__": "SGDOptimizer", + "scaler:__choice__": "StandardScaler", + "trainer:__choice__": "MixUpTrainer", + "feature_preprocessor:Nystroem:kernel": "cosine", + "feature_preprocessor:Nystroem:n_components": 4, + "lr_scheduler:StepLR:gamma": 0.5071782128761503, + "lr_scheduler:StepLR:step_size": 9, + "network_backbone:MLPBackbone:activation": "tanh", + "network_backbone:MLPBackbone:num_groups": 12, + "network_backbone:MLPBackbone:num_units_1": 696, + "network_backbone:MLPBackbone:use_dropout": false, + "network_head:fully_connected:num_layers": 2, + "network_init:KaimingInit:bias_strategy": "Normal", + "optimizer:SGDOptimizer:lr": 0.005882358405127406, + "optimizer:SGDOptimizer:momentum": 0.44948646190205266, + "optimizer:SGDOptimizer:weight_decay": 0.042125609913750056, + "trainer:MixUpTrainer:alpha": 0.5745324875812398, + "trainer:MixUpTrainer:weighted_loss": false, + "network_backbone:MLPBackbone:num_units_10": 819, + "network_backbone:MLPBackbone:num_units_11": 520, + "network_backbone:MLPBackbone:num_units_12": 498, + "network_backbone:MLPBackbone:num_units_2": 687, + "network_backbone:MLPBackbone:num_units_3": 160, + "network_backbone:MLPBackbone:num_units_4": 751, + "network_backbone:MLPBackbone:num_units_5": 934, + "network_backbone:MLPBackbone:num_units_6": 165, + "network_backbone:MLPBackbone:num_units_7": 848, + "network_backbone:MLPBackbone:num_units_8": 757, + "network_backbone:MLPBackbone:num_units_9": 605, + "network_head:fully_connected:activation": "sigmoid", + "network_head:fully_connected:units_layer_1": 191 + }, + "8": { + "data_loader:batch_size": 130, + "encoder:__choice__": "NoEncoder", + "feature_preprocessor:__choice__": "PolynomialFeatures", + "imputer:numerical_strategy": "median", + "lr_scheduler:__choice__": "CyclicLR", + "network_backbone:__choice__": "ResNetBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "SparseInit", + "optimizer:__choice__": "AdamOptimizer", + "scaler:__choice__": "StandardScaler", + "trainer:__choice__": "StandardTrainer", + "feature_preprocessor:PolynomialFeatures:degree": 3, + "feature_preprocessor:PolynomialFeatures:include_bias": true, + "feature_preprocessor:PolynomialFeatures:interaction_only": false, + "lr_scheduler:CyclicLR:base_lr": 0.04071276269672806, + "lr_scheduler:CyclicLR:max_lr": 0.02877849608326783, + "lr_scheduler:CyclicLR:mode": "triangular", + "lr_scheduler:CyclicLR:step_size_up": 3470, + "network_backbone:ResNetBackbone:activation": "sigmoid", + "network_backbone:ResNetBackbone:blocks_per_group_0": 4, + "network_backbone:ResNetBackbone:blocks_per_group_1": 4, + "network_backbone:ResNetBackbone:num_groups": 8, + "network_backbone:ResNetBackbone:num_units_0": 854, + "network_backbone:ResNetBackbone:num_units_1": 973, + "network_backbone:ResNetBackbone:use_dropout": false, + "network_backbone:ResNetBackbone:use_shake_drop": false, + "network_backbone:ResNetBackbone:use_shake_shake": true, + "network_head:fully_connected:num_layers": 2, + "network_init:SparseInit:bias_strategy": "Normal", + "optimizer:AdamOptimizer:beta1": 0.9072214493192293, + "optimizer:AdamOptimizer:beta2": 0.900926929525363, + "optimizer:AdamOptimizer:lr": 0.03886692855998248, + "optimizer:AdamOptimizer:weight_decay": 0.01179277956957403, + "trainer:StandardTrainer:weighted_loss": false, + "network_backbone:ResNetBackbone:blocks_per_group_2": 2, + "network_backbone:ResNetBackbone:blocks_per_group_3": 3, + "network_backbone:ResNetBackbone:blocks_per_group_4": 4, + "network_backbone:ResNetBackbone:blocks_per_group_5": 3, + "network_backbone:ResNetBackbone:blocks_per_group_6": 2, + "network_backbone:ResNetBackbone:blocks_per_group_7": 1, + "network_backbone:ResNetBackbone:blocks_per_group_8": 2, + "network_backbone:ResNetBackbone:num_units_2": 995, + "network_backbone:ResNetBackbone:num_units_3": 680, + "network_backbone:ResNetBackbone:num_units_4": 196, + "network_backbone:ResNetBackbone:num_units_5": 147, + "network_backbone:ResNetBackbone:num_units_6": 748, + "network_backbone:ResNetBackbone:num_units_7": 936, + "network_backbone:ResNetBackbone:num_units_8": 870, + "network_head:fully_connected:activation": "tanh", + "network_head:fully_connected:units_layer_1": 425 + }, + "9": { + "data_loader:batch_size": 137, + "encoder:__choice__": "NoEncoder", + "feature_preprocessor:__choice__": "Nystroem", + "imputer:numerical_strategy": "mean", + "lr_scheduler:__choice__": "CosineAnnealingLR", + "network_backbone:__choice__": "ResNetBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "KaimingInit", + "optimizer:__choice__": "RMSpropOptimizer", + "scaler:__choice__": "Normalizer", + "trainer:__choice__": "StandardTrainer", + "feature_preprocessor:Nystroem:kernel": "rbf", + "feature_preprocessor:Nystroem:n_components": 3, + "lr_scheduler:CosineAnnealingLR:T_max": 225, + "network_backbone:ResNetBackbone:activation": "tanh", + "network_backbone:ResNetBackbone:blocks_per_group_0": 4, + "network_backbone:ResNetBackbone:blocks_per_group_1": 2, + "network_backbone:ResNetBackbone:num_groups": 6, + "network_backbone:ResNetBackbone:num_units_0": 750, + "network_backbone:ResNetBackbone:num_units_1": 401, + "network_backbone:ResNetBackbone:use_dropout": false, + "network_backbone:ResNetBackbone:use_shake_drop": false, + "network_backbone:ResNetBackbone:use_shake_shake": false, + "network_head:fully_connected:num_layers": 2, + "network_init:KaimingInit:bias_strategy": "Zero", + "optimizer:RMSpropOptimizer:alpha": 0.7538231500569799, + "optimizer:RMSpropOptimizer:lr": 1.1725416565679341e-05, + "optimizer:RMSpropOptimizer:momentum": 0.4213512475609884, + "optimizer:RMSpropOptimizer:weight_decay": 0.005044227636227261, + "scaler:Normalizer:norm": "mean_abs", + "trainer:StandardTrainer:weighted_loss": false, + "feature_preprocessor:Nystroem:gamma": 0.0051035411586294694, + "network_backbone:ResNetBackbone:blocks_per_group_2": 1, + "network_backbone:ResNetBackbone:blocks_per_group_3": 1, + "network_backbone:ResNetBackbone:blocks_per_group_4": 3, + "network_backbone:ResNetBackbone:blocks_per_group_5": 3, + "network_backbone:ResNetBackbone:blocks_per_group_6": 1, + "network_backbone:ResNetBackbone:num_units_2": 1018, + "network_backbone:ResNetBackbone:num_units_3": 501, + "network_backbone:ResNetBackbone:num_units_4": 911, + "network_backbone:ResNetBackbone:num_units_5": 991, + "network_backbone:ResNetBackbone:num_units_6": 584, + "network_head:fully_connected:activation": "tanh", + "network_head:fully_connected:units_layer_1": 294 + } + } +} \ No newline at end of file diff --git a/test/test_api/test_api.py b/test/test_api/test_api.py index 3954be31a..d2a131c24 100644 --- a/test/test_api/test_api.py +++ b/test/test_api/test_api.py @@ -8,8 +8,6 @@ from ConfigSpace.configuration_space import Configuration -from api_utils import dummy_traditional_classification - import numpy as np import pandas as pd From af0480420a7140d93aef5a451ad3b835acdd6db3 Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Mon, 17 May 2021 23:33:13 +0200 Subject: [PATCH 11/16] fix flake --- test/test_api/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_api/utils.py b/test/test_api/utils.py index d68f26dfa..d09f66c9a 100644 --- a/test/test_api/utils.py +++ b/test/test_api/utils.py @@ -1,6 +1,6 @@ import os -from smac.runhistory.runhistory import RunHistory, DataOrigin +from smac.runhistory.runhistory import DataOrigin, RunHistory from autoPyTorch.constants import REGRESSION_TASKS from autoPyTorch.evaluation.abstract_evaluator import ( From ae7b29619ab047ec69ea55aab93d1ba67bc71815 Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Fri, 21 May 2021 14:04:12 +0200 Subject: [PATCH 12/16] Added debug information for API --- test/test_api/test_api.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test/test_api/test_api.py b/test/test_api/test_api.py index d2a131c24..95c6b9595 100644 --- a/test/test_api/test_api.py +++ b/test/test_api/test_api.py @@ -197,7 +197,8 @@ def test_tabular_classification(openml_id, resampling_strategy, backend, resampl incumbent_config, incumbent_results = estimator.get_incumbent_results() assert isinstance(incumbent_config, Configuration) assert isinstance(incumbent_results, dict) - assert 'opt_loss' in incumbent_results + assert 'opt_loss' in incumbent_results, "run history: {}, successful_num_run: {}".format(estimator.run_history.data, + successful_num_run) assert 'train_loss' in incumbent_results # Check that we can pickle @@ -265,7 +266,7 @@ def test_tabular_regression(openml_name, resampling_strategy, backend, resamplin X_test=X_test, y_test=y_test, optimize_metric='r2', total_walltime_limit=30, - func_eval_time_limit_secs=5, + func_eval_time_limit_secs=10, enable_traditional_pipeline=False, ) @@ -379,8 +380,9 @@ def test_tabular_regression(openml_name, resampling_strategy, backend, resamplin incumbent_config, incumbent_results = estimator.get_incumbent_results() assert isinstance(incumbent_config, Configuration) assert isinstance(incumbent_results, dict) - assert 'opt_loss' in incumbent_results - assert 'train_loss' in incumbent_results + assert 'opt_loss' in incumbent_results, "run history: {}, successful_num_run: {}".format(estimator.run_history.data, + successful_num_run) + assert 'train_loss' in incumbent_results, estimator.run_history.data # Check that we can pickle # Test pickle From 5fd93cb5be5a580cb83d1726ce42306fe1b4881b Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Fri, 21 May 2021 16:43:17 +0200 Subject: [PATCH 13/16] filtering only successful runs in get_incumbent_results --- autoPyTorch/api/base_task.py | 2 ++ test/test_api/test_api.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/autoPyTorch/api/base_task.py b/autoPyTorch/api/base_task.py index 3160cdcd9..30c3c676b 100644 --- a/autoPyTorch/api/base_task.py +++ b/autoPyTorch/api/base_task.py @@ -1283,6 +1283,8 @@ def get_incumbent_results( filter(lambda elem: elem[1].additional_info is not None and elem[1]. additional_info['configuration_origin'] != 'traditional', run_history_data.items())) + run_history_data = dict( + filter(lambda elem: 'SUCCESS' in str(elem[1].status), run_history_data.items())) sorted_runvalue_by_cost = sorted(run_history_data.items(), key=lambda item: item[1].cost) incumbent_run_key, incumbent_run_value = sorted_runvalue_by_cost[0] incumbent_config = self.run_history.ids_config[incumbent_run_key.config_id] diff --git a/test/test_api/test_api.py b/test/test_api/test_api.py index 95c6b9595..cb09be0eb 100644 --- a/test/test_api/test_api.py +++ b/test/test_api/test_api.py @@ -266,7 +266,7 @@ def test_tabular_regression(openml_name, resampling_strategy, backend, resamplin X_test=X_test, y_test=y_test, optimize_metric='r2', total_walltime_limit=30, - func_eval_time_limit_secs=10, + func_eval_time_limit_secs=5, enable_traditional_pipeline=False, ) From e0c3a3b799fffc3351f93b8980128820cecf9848 Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Tue, 25 May 2021 11:48:39 +0200 Subject: [PATCH 14/16] Address comments from fransisco --- autoPyTorch/api/base_task.py | 3 +-- autoPyTorch/evaluation/train_evaluator.py | 5 ++++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/autoPyTorch/api/base_task.py b/autoPyTorch/api/base_task.py index 30c3c676b..85246b4ca 100644 --- a/autoPyTorch/api/base_task.py +++ b/autoPyTorch/api/base_task.py @@ -1271,8 +1271,7 @@ def get_incumbent_results( Returns: """ - assert self.run_history is not None, "No Run History found, search has not been called." - if self.run_history.empty(): + if self.run_history is not None or self.run_history.empty(): raise ValueError("Run History is empty. Something went wrong, " "smac was not able to fit any model?") diff --git a/autoPyTorch/evaluation/train_evaluator.py b/autoPyTorch/evaluation/train_evaluator.py index 2c5459beb..7ecd5249f 100644 --- a/autoPyTorch/evaluation/train_evaluator.py +++ b/autoPyTorch/evaluation/train_evaluator.py @@ -173,7 +173,10 @@ def fit_predict_and_loss(self) -> None: status = StatusType.SUCCESS self.logger.debug("In train evaluator fit_predict_and_loss, num_run: {} loss:{}," - " additional run info:{}".format(self.num_run, loss, additional_run_info)) + " additional run info:{}, status: {}".format(self.num_run, + loss, + additional_run_info, + status)) self.finish_up( loss=loss, train_loss=train_loss, From 60a744d0323a91030790f22bab3732c15361adb7 Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Tue, 25 May 2021 12:11:56 +0200 Subject: [PATCH 15/16] Revert changes made to run history assertion in base taks #1257 --- autoPyTorch/api/base_task.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/autoPyTorch/api/base_task.py b/autoPyTorch/api/base_task.py index 85246b4ca..30c3c676b 100644 --- a/autoPyTorch/api/base_task.py +++ b/autoPyTorch/api/base_task.py @@ -1271,7 +1271,8 @@ def get_incumbent_results( Returns: """ - if self.run_history is not None or self.run_history.empty(): + assert self.run_history is not None, "No Run History found, search has not been called." + if self.run_history.empty(): raise ValueError("Run History is empty. Something went wrong, " "smac was not able to fit any model?") From fa4288a045c2a2b78950415715ae78b574e15689 Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Mon, 31 May 2021 16:17:05 +0200 Subject: [PATCH 16/16] fix flake issue --- test/test_api/test_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_api/test_api.py b/test/test_api/test_api.py index cb09be0eb..aeab572a5 100644 --- a/test/test_api/test_api.py +++ b/test/test_api/test_api.py @@ -558,7 +558,7 @@ def test_portfolio_selection_failure(openml_id, backend, n_samples): estimator = TabularClassificationTask( backend=backend, resampling_strategy=HoldoutValTypes.holdout_validation, - include_components = include + include_components=include ) with pytest.raises(FileNotFoundError, match=r"The path: .+? provided for 'portfolio_selection' " r"for the file containing the portfolio configurations "