From dcb3ca1345f21bec930d9c14661d321bfb3851ef Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Mon, 8 Aug 2022 17:21:09 +0200 Subject: [PATCH 1/4] add flexibility to avoid checking for test scores --- autoPyTorch/utils/results_manager.py | 57 +- autoPyTorch/utils/results_visualizer.py | 10 +- test/test_utils/runhistory_no_test.json | 1582 ++++++++++++++++++++ test/test_utils/test_results_manager.py | 53 + test/test_utils/test_results_visualizer.py | 36 + 5 files changed, 1733 insertions(+), 5 deletions(-) create mode 100644 test/test_utils/runhistory_no_test.json diff --git a/autoPyTorch/utils/results_manager.py b/autoPyTorch/utils/results_manager.py index c1860b0f6..a374cd2bc 100644 --- a/autoPyTorch/utils/results_manager.py +++ b/autoPyTorch/utils/results_manager.py @@ -28,6 +28,9 @@ ] +OPTIONAL_INFERENCE_CHOICES = ('test',) + + def cost2metric(cost: float, metric: autoPyTorchMetric) -> float: """ Revert cost metric evaluated in SMAC to the original metric. @@ -97,7 +100,14 @@ def _extract_metrics_info( if inference_name not in inference_choices: raise ValueError(f'inference_name must be in {inference_choices}, but got {inference_choices}') - cost_info = run_value.additional_info[f'{inference_name}_loss'] + cost_info = run_value.additional_info.get(f'{inference_name}_loss', None) + if cost_info is None: + if inference_name not in OPTIONAL_INFERENCE_CHOICES: + raise ValueError(f"Expected loss for {inference_name} set to not be None, but got {cost_info}") + else: + # Additional info for metrics is not available in this case. + return {metric.name: None for metric in scoring_functions} + avail_metrics = cost_info.keys() return { @@ -175,7 +185,7 @@ def _update(self, data: Dict[str, Any]) -> None: ) self._train_scores.append(data[f'train_{self.metric_name}']) - self._test_scores.append(data[f'test_{self.metric_name}']) + self._test_scores.append(data.get(f'test_{self.metric_name}', None)) self._end_times.append(datetime.timestamp(data['Timestamp'])) def _sort_by_endtime(self) -> None: @@ -413,11 +423,31 @@ def _extract_results_from_run_history(self, run_history: RunHistory) -> None: config = run_history.ids_config[run_key.config_id] self._update(config=config, run_key=run_key, run_value=run_value) + self._check_null_in_optional_inference_choices() + self.rank_opt_scores = scipy.stats.rankdata( -1 * self._metric._sign * self.opt_scores, # rank order method='min' ) + def _check_null_in_optional_inference_choices( + self + ) -> None: + """ + Checks if the data is missing for each optional inference choice and + sets the scores for that inference choice to all None. + """ + for inference_choice in OPTIONAL_INFERENCE_CHOICES: + metrics_dict = getattr(self, f'{inference_choice}_metric_dict') + new_metric_dict = {} + + for metric in self._scoring_functions: + scores = metrics_dict[metric.name] + if all([score is None or score == metric._worst_possible_result for score in scores]): + scores = [None] * len(self.status_types) + new_metric_dict[metric.name] = scores + setattr(self, f'{inference_choice}_metric_dict', new_metric_dict) + class MetricResults: def __init__( @@ -486,12 +516,22 @@ def _extract_results(self) -> None: for inference_name in ['train', 'test', 'opt']: # TODO: Extract information from self.search_results data = getattr(self.search_results, f'{inference_name}_metric_dict')[metric_name] + if all([d is None for d in data]): + if inference_name not in OPTIONAL_INFERENCE_CHOICES: + raise ValueError(f"Expected {metric_name} score for {inference_name} set to not be None, but got {data}") + else: + continue self.data[f'single::{inference_name}::{metric_name}'] = np.array(data) if self.ensemble_results.empty() or inference_name == 'opt': continue data = getattr(self.ensemble_results, f'{inference_name}_scores') + if all([d is None for d in data]): + if inference_name not in OPTIONAL_INFERENCE_CHOICES: + raise ValueError(f"Expected {metric_name} score for {inference_name} set to not be None, but got {data}") + else: + continue self.data[f'ensemble::{inference_name}::{metric_name}'] = np.array(data) def get_ensemble_merged_data(self) -> Dict[str, np.ndarray]: @@ -516,6 +556,8 @@ def get_ensemble_merged_data(self) -> Dict[str, np.ndarray]: cur, timestep_size, sign = 0, self.cum_times.size, self.metric._sign key_train, key_test = f'ensemble::train::{self.metric.name}', f'ensemble::test::{self.metric.name}' + all_test_perfs_null = all([[perf is None for perf in test_scores]]) + train_perfs = np.full_like(self.cum_times, self.metric._worst_possible_result) test_perfs = np.full_like(self.cum_times, self.metric._worst_possible_result) @@ -530,9 +572,16 @@ def get_ensemble_merged_data(self) -> Dict[str, np.ndarray]: time_index = min(cur, timestep_size - 1) # If there already exists a previous allocated value, update by a better value train_perfs[time_index] = sign * max(sign * train_perfs[time_index], sign * train_score) - test_perfs[time_index] = sign * max(sign * test_perfs[time_index], sign * test_score) + # test_perfs can be none when X_test is not passed + if not all_test_perfs_null: + test_perfs[time_index] = sign * max(sign * test_perfs[time_index], sign * test_score) + + update_dict = {key_train: train_perfs} + if not all_test_perfs_null: + update_dict[key_test] = test_perfs + + data.update(update_dict) - data.update({key_train: train_perfs, key_test: test_perfs}) return data diff --git a/autoPyTorch/utils/results_visualizer.py b/autoPyTorch/utils/results_visualizer.py index e1debe29c..9853ab7f9 100644 --- a/autoPyTorch/utils/results_visualizer.py +++ b/autoPyTorch/utils/results_visualizer.py @@ -6,7 +6,7 @@ import numpy as np -from autoPyTorch.utils.results_manager import MetricResults +from autoPyTorch.utils.results_manager import OPTIONAL_INFERENCE_CHOICES, MetricResults plt.rcParams["font.family"] = "Times New Roman" @@ -318,7 +318,15 @@ def plot_perf_over_time( minimize = (results.metric._sign == -1) for key in data.keys(): + inference_name = key.split('::')[1] _label, _color, _perfs = labels[key], colors[key], data[key] + all_null_perfs = all([perf is None for perf in _perfs]) + + if all_null_perfs: + if inference_name not in OPTIONAL_INFERENCE_CHOICES: + raise ValueError(f"Expected loss for {inference_name} set to not be None") + else: + continue # Take the best results over time _cum_perfs = np.minimum.accumulate(_perfs) if minimize else np.maximum.accumulate(_perfs) diff --git a/test/test_utils/runhistory_no_test.json b/test/test_utils/runhistory_no_test.json new file mode 100644 index 000000000..35bf7311b --- /dev/null +++ b/test/test_utils/runhistory_no_test.json @@ -0,0 +1,1582 @@ +{ + "data": [ + [ + [ + 1, + "{\"task_id\": \"Australian\"}", + 0, + 5.555555555555555 + ], + [ + 0.15204678362573099, + 3.154788017272949, + { + "__enum__": "StatusType.SUCCESS" + }, + 1637342638.6119366, + 1637342642.7887495, + { + "opt_loss": { + "accuracy": 0.15204678362573099, + "balanced_accuracy": 0.15263157894736845, + "roc_auc": 0.08981994459833786, + "average_precision": 0.1040861796433199, + "log_loss": 0.5765479137672738, + "precision": 0.17948717948717952, + "precision_macro": 0.15425971877584788, + "precision_micro": 0.15204678362573099, + "precision_weighted": 0.15145666758569976, + "recall": 0.1578947368421053, + "recall_macro": 0.15263157894736845, + "recall_micro": 0.15204678362573099, + "recall_weighted": 0.15204678362573099, + "f1": 0.16883116883116878, + "f1_macro": 0.15356452058579717, + "f1_micro": 0.15204678362573099, + "f1_weighted": 0.15186822633631147 + }, + "duration": 3.11077618598938, + "num_run": 8, + "train_loss": { + "accuracy": 0.09537572254335258, + "balanced_accuracy": 0.10239948774980623, + "roc_auc": 0.03963198867657458, + "average_precision": 0.044469547423341305, + "log_loss": 0.28008669264774966, + "precision": 0.03731343283582089, + "precision_macro": 0.08469445226696704, + "precision_micro": 0.09537572254335258, + "precision_weighted": 0.0890765118675354, + "recall": 0.17834394904458595, + "recall_macro": 0.10239948774980623, + "recall_micro": 0.09537572254335258, + "recall_weighted": 0.09537572254335258, + "f1": 0.11340206185567014, + "f1_macro": 0.09784816309741107, + "f1_micro": 0.09537572254335258, + "f1_weighted": 0.0964096522295953 + }, + "configuration_origin": "Default" + } + ] + ], + [ + [ + 2, + "{\"task_id\": \"Australian\"}", + 0, + 5.555555555555555 + ], + [ + 0.4444444444444444, + 3.2763524055480957, + { + "__enum__": "StatusType.SUCCESS" + }, + 1637342642.963385, + 1637342647.2651122, + { + "opt_loss": { + "accuracy": 0.4444444444444444, + "balanced_accuracy": 0.5, + "roc_auc": 0.25526315789473697, + "average_precision": 0.35005634879129066, + "log_loss": 1.0913122792494052, + "precision": 1.0, + "precision_macro": 0.7222222222222222, + "precision_micro": 0.4444444444444444, + "precision_weighted": 0.691358024691358, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.4444444444444444, + "recall_weighted": 0.4444444444444444, + "f1": 1.0, + "f1_macro": 0.6428571428571428, + "f1_micro": 0.4444444444444444, + "f1_weighted": 0.6031746031746031 + }, + "duration": 3.2138161659240723, + "num_run": 9, + "train_loss": { + "accuracy": 0.45375722543352603, + "balanced_accuracy": 0.5, + "roc_auc": 0.2745256630606949, + "average_precision": 0.4037230365622788, + "log_loss": 1.1229484684905306, + "precision": 1.0, + "precision_macro": 0.726878612716763, + "precision_micro": 0.45375722543352603, + "precision_weighted": 0.7016188312339203, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.45375722543352603, + "recall_weighted": 0.45375722543352603, + "f1": 1.0, + "f1_macro": 0.6467289719626168, + "f1_micro": 0.45375722543352603, + "f1_weighted": 0.6140565069418183 + }, + "configuration_origin": "Random Search" + } + ] + ], + [ + [ + 3, + "{\"task_id\": \"Australian\"}", + 0, + 5.555555555555555 + ], + [ + 0.5555555555555556, + 22.723600149154663, + { + "__enum__": "StatusType.SUCCESS" + }, + 1637342651.4707444, + 1637342675.2555833, + { + "opt_loss": { + "accuracy": 0.5555555555555556, + "balanced_accuracy": 0.5, + "roc_auc": 0.4924515235457063, + "average_precision": 0.5493808049535605, + "log_loss": 0.7291908971747459, + "precision": 0.5555555555555556, + "precision_macro": 0.7777777777777778, + "precision_micro": 0.5555555555555556, + "precision_weighted": 0.8024691358024691, + "recall": 0.0, + "recall_macro": 0.5, + "recall_micro": 0.5555555555555556, + "recall_weighted": 0.5555555555555556, + "f1": 0.3846153846153847, + "f1_macro": 0.6923076923076923, + "f1_micro": 0.5555555555555556, + "f1_weighted": 0.7264957264957266 + }, + "duration": 22.021637201309204, + "num_run": 10, + "train_loss": { + "accuracy": 0.546242774566474, + "balanced_accuracy": 0.5, + "roc_auc": 0.514423887035352, + "average_precision": 0.5521926852639938, + "log_loss": 0.7258427792546377, + "precision": 0.546242774566474, + "precision_macro": 0.773121387283237, + "precision_micro": 0.546242774566474, + "precision_weighted": 0.7941043803668683, + "recall": 0.0, + "recall_macro": 0.5, + "recall_micro": 0.546242774566474, + "recall_weighted": 0.546242774566474, + "f1": 0.3757455268389662, + "f1_macro": 0.6878727634194831, + "f1_micro": 0.546242774566474, + "f1_weighted": 0.7167400222939817 + }, + "configuration_origin": "Random Search (sorted)" + } + ] + ], + [ + [ + 4, + "{\"task_id\": \"Australian\"}", + 0, + 5.555555555555555 + ], + [ + 0.29824561403508776, + 4.990685224533081, + { + "__enum__": "StatusType.SUCCESS" + }, + 1637342675.317421, + 1637342681.334954, + { + "opt_loss": { + "accuracy": 0.29824561403508776, + "balanced_accuracy": 0.30263157894736836, + "roc_auc": 0.26869806094182835, + "average_precision": 0.3191125709864897, + "log_loss": 0.6374789248084465, + "precision": 0.33333333333333337, + "precision_macro": 0.30208333333333337, + "precision_micro": 0.29824561403508776, + "precision_weighted": 0.29861111111111116, + "recall": 0.3421052631578947, + "recall_macro": 0.30263157894736836, + "recall_micro": 0.29824561403508776, + "recall_weighted": 0.29824561403508776, + "f1": 0.3377483443708609, + "f1_macro": 0.30238202558857186, + "f1_micro": 0.29824561403508776, + "f1_weighted": 0.29845243461276183 + }, + "duration": 4.924501419067383, + "num_run": 11, + "train_loss": { + "accuracy": 0.3728323699421965, + "balanced_accuracy": 0.3800930138509757, + "roc_auc": 0.3314460957773059, + "average_precision": 0.3638537658311296, + "log_loss": 0.6533903728503023, + "precision": 0.4014084507042254, + "precision_macro": 0.3771748135874068, + "precision_micro": 0.3728323699421965, + "precision_weighted": 0.3749335523511692, + "recall": 0.4585987261146497, + "recall_macro": 0.3800930138509757, + "recall_micro": 0.3728323699421965, + "recall_weighted": 0.3728323699421965, + "f1": 0.43143812709030094, + "f1_macro": 0.3798412009497306, + "f1_micro": 0.3728323699421965, + "f1_weighted": 0.3750692309020478 + }, + "configuration_origin": "Random Search" + } + ] + ], + [ + [ + 5, + "{\"task_id\": \"Australian\"}", + 0, + 5.555555555555555 + ], + [ + 0.4444444444444444, + 10.684926509857178, + { + "__enum__": "StatusType.SUCCESS" + }, + 1637342681.548915, + 1637342693.2717755, + { + "opt_loss": { + "accuracy": 0.4444444444444444, + "balanced_accuracy": 0.5, + "roc_auc": 0.6092797783933518, + "average_precision": 0.6129755132627962, + "log_loss": 0.6905045174715811, + "precision": 1.0, + "precision_macro": 0.7222222222222222, + "precision_micro": 0.4444444444444444, + "precision_weighted": 0.691358024691358, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.4444444444444444, + "recall_weighted": 0.4444444444444444, + "f1": 1.0, + "f1_macro": 0.6428571428571428, + "f1_micro": 0.4444444444444444, + "f1_weighted": 0.6031746031746031 + }, + "duration": 10.401196956634521, + "num_run": 12, + "train_loss": { + "accuracy": 0.45375722543352603, + "balanced_accuracy": 0.5, + "roc_auc": 0.6309102551140767, + "average_precision": 0.6325768698403712, + "log_loss": 0.691941062839045, + "precision": 1.0, + "precision_macro": 0.726878612716763, + "precision_micro": 0.45375722543352603, + "precision_weighted": 0.7016188312339203, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.45375722543352603, + "recall_weighted": 0.45375722543352603, + "f1": 1.0, + "f1_macro": 0.6467289719626168, + "f1_micro": 0.45375722543352603, + "f1_weighted": 0.6140565069418183 + }, + "configuration_origin": "Random Search" + } + ] + ], + [ + [ + 6, + "{\"task_id\": \"Australian\"}", + 0, + 5.555555555555555 + ], + [ + 0.4444444444444444, + 9.947429180145264, + { + "__enum__": "StatusType.SUCCESS" + }, + 1637342693.356699, + 1637342704.341065, + { + "opt_loss": { + "accuracy": 0.4444444444444444, + "balanced_accuracy": 0.5, + "roc_auc": 0.24930747922437668, + "average_precision": 0.31612650360994055, + "log_loss": 0.6525155201292875, + "precision": 1.0, + "precision_macro": 0.7222222222222222, + "precision_micro": 0.4444444444444444, + "precision_weighted": 0.691358024691358, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.4444444444444444, + "recall_weighted": 0.4444444444444444, + "f1": 1.0, + "f1_macro": 0.6428571428571428, + "f1_micro": 0.4444444444444444, + "f1_weighted": 0.6031746031746031 + }, + "duration": 9.76927137374878, + "num_run": 13, + "train_loss": { + "accuracy": 0.45375722543352603, + "balanced_accuracy": 0.5, + "roc_auc": 0.22427796313146642, + "average_precision": 0.2451792573360162, + "log_loss": 0.64721482587343, + "precision": 1.0, + "precision_macro": 0.726878612716763, + "precision_micro": 0.45375722543352603, + "precision_weighted": 0.7016188312339203, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.45375722543352603, + "recall_weighted": 0.45375722543352603, + "f1": 1.0, + "f1_macro": 0.6467289719626168, + "f1_micro": 0.45375722543352603, + "f1_weighted": 0.6140565069418183 + }, + "configuration_origin": "Random Search" + } + ] + ], + [ + [ + 7, + "{\"task_id\": \"Australian\"}", + 0, + 5.555555555555555 + ], + [ + 1.0, + 11.687273979187012, + { + "__enum__": "StatusType.CRASHED" + }, + 1637342713.4931705, + 1637342726.1866672, + { + "error": "Result queue is empty", + "exit_status": "", + "subprocess_stdout": "", + "subprocess_stderr": "Process pynisher function call:\nTraceback (most recent call last):\n File \"/home/shuhei/research/Auto-PyTorch/autoPyTorch/evaluation/tae.py\", line 39, in fit_predict_try_except_decorator\n ta(queue=queue, **kwargs)\n File \"/home/shuhei/research/Auto-PyTorch/autoPyTorch/evaluation/train_evaluator.py\", line 485, in eval_function\n evaluator.fit_predict_and_loss()\n File \"/home/shuhei/research/Auto-PyTorch/autoPyTorch/evaluation/train_evaluator.py\", line 163, in fit_predict_and_loss\n y_train_pred, y_opt_pred, y_valid_pred, y_test_pred = self._fit_and_predict(pipeline, split_id,\n File \"/home/shuhei/research/Auto-PyTorch/autoPyTorch/evaluation/train_evaluator.py\", line 337, in _fit_and_predict\n fit_and_suppress_warnings(self.logger, pipeline, X, y)\n File \"/home/shuhei/research/Auto-PyTorch/autoPyTorch/evaluation/abstract_evaluator.py\", line 321, in fit_and_suppress_warnings\n pipeline.fit(X, y)\n File \"/home/shuhei/research/Auto-PyTorch/autoPyTorch/pipeline/base_pipeline.py\", line 153, in fit\n self.fit_estimator(X, y, **fit_params)\n File \"/home/shuhei/research/Auto-PyTorch/autoPyTorch/pipeline/base_pipeline.py\", line 172, in fit_estimator\n self._final_estimator.fit(X, y, **fit_params)\n File \"/home/shuhei/research/Auto-PyTorch/autoPyTorch/pipeline/components/training/trainer/__init__.py\", line 211, in fit\n self._fit(\n File \"/home/shuhei/research/Auto-PyTorch/autoPyTorch/pipeline/components/training/trainer/__init__.py\", line 290, in _fit\n train_loss, train_metrics = self.choice.train_epoch(\n File \"/home/shuhei/research/Auto-PyTorch/autoPyTorch/pipeline/components/training/trainer/base_trainer.py\", line 303, in train_epoch\n loss, outputs = self.train_step(data, targets)\n File \"/home/shuhei/research/Auto-PyTorch/autoPyTorch/pipeline/components/training/trainer/base_trainer.py\", line 357, in train_step\n outputs = self.model(data)\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 727, in _call_impl\n result = self.forward(*input, **kwargs)\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/site-packages/torch/nn/modules/container.py\", line 117, in forward\n input = module(input)\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 727, in _call_impl\n result = self.forward(*input, **kwargs)\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/site-packages/torch/nn/modules/container.py\", line 117, in forward\n input = module(input)\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 727, in _call_impl\n result = self.forward(*input, **kwargs)\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/site-packages/torch/nn/modules/container.py\", line 117, in forward\n input = module(input)\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 727, in _call_impl\n result = self.forward(*input, **kwargs)\n File \"/home/shuhei/research/Auto-PyTorch/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py\", line 274, in forward\n x2 = self.shake_shake_layers(x)\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 727, in _call_impl\n result = self.forward(*input, **kwargs)\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/site-packages/torch/nn/modules/container.py\", line 117, in forward\n input = module(input)\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 727, in _call_impl\n result = self.forward(*input, **kwargs)\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/site-packages/torch/nn/modules/linear.py\", line 93, in forward\n return F.linear(input, self.weight, self.bias)\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/site-packages/torch/nn/functional.py\", line 1690, in linear\n ret = torch.addmm(bias, input, weight.t())\nRuntimeError: [enforce fail at CPUAllocator.cpp:65] . DefaultCPUAllocator: can't allocate memory: you tried to allocate 713632 bytes. Error code 12 (Cannot allocate memory)\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/multiprocessing/process.py\", line 315, in _bootstrap\n self.run()\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/multiprocessing/process.py\", line 108, in run\n self._target(*self._args, **self._kwargs)\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/site-packages/pynisher/limit_function_call.py\", line 138, in subprocess_func\n return_value = ((func(*args, **kwargs), 0))\n File \"/home/shuhei/research/Auto-PyTorch/autoPyTorch/evaluation/tae.py\", line 52, in fit_predict_try_except_decorator\n queue.put({'loss': cost_for_crash,\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/multiprocessing/queues.py\", line 88, in put\n self._start_thread()\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/multiprocessing/queues.py\", line 173, in _start_thread\n self._thread.start()\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/threading.py\", line 852, in start\n _start_new_thread(self._bootstrap, ())\nRuntimeError: can't start new thread\n", + "exitcode": 1, + "configuration_origin": "Random Search (sorted)" + } + ] + ], + [ + [ + 8, + "{\"task_id\": \"Australian\"}", + 0, + 5.555555555555555 + ], + [ + 0.5555555555555556, + 8.478890419006348, + { + "__enum__": "StatusType.SUCCESS" + }, + 1637342733.815657, + 1637342743.3274522, + { + "opt_loss": { + "accuracy": 0.5555555555555556, + "balanced_accuracy": 0.5, + "roc_auc": 0.44605263157894737, + "average_precision": 0.526907034743722, + "log_loss": 0.722785997111895, + "precision": 0.5555555555555556, + "precision_macro": 0.7777777777777778, + "precision_micro": 0.5555555555555556, + "precision_weighted": 0.8024691358024691, + "recall": 0.0, + "recall_macro": 0.5, + "recall_micro": 0.5555555555555556, + "recall_weighted": 0.5555555555555556, + "f1": 0.3846153846153847, + "f1_macro": 0.6923076923076923, + "f1_micro": 0.5555555555555556, + "f1_weighted": 0.7264957264957266 + }, + "duration": 8.288825988769531, + "num_run": 15, + "train_loss": { + "accuracy": 0.546242774566474, + "balanced_accuracy": 0.5, + "roc_auc": 0.4537121288713646, + "average_precision": 0.5218043063878082, + "log_loss": 0.7198673617633092, + "precision": 0.546242774566474, + "precision_macro": 0.773121387283237, + "precision_micro": 0.546242774566474, + "precision_weighted": 0.7941043803668683, + "recall": 0.0, + "recall_macro": 0.5, + "recall_micro": 0.546242774566474, + "recall_weighted": 0.546242774566474, + "f1": 0.3757455268389662, + "f1_macro": 0.6878727634194831, + "f1_micro": 0.546242774566474, + "f1_weighted": 0.7167400222939817 + }, + "configuration_origin": "Random Search (sorted)" + } + ] + ], + [ + [ + 9, + "{\"task_id\": \"Australian\"}", + 0, + 5.555555555555555 + ], + [ + 0.4444444444444444, + 5.485020637512207, + { + "__enum__": "StatusType.SUCCESS" + }, + 1637342743.4267018, + 1637342749.9442234, + { + "opt_loss": { + "accuracy": 0.4444444444444444, + "balanced_accuracy": 0.5, + "roc_auc": 0.5, + "average_precision": 0.5555555555555556, + "log_loss": 15.350567287868923, + "precision": 1.0, + "precision_macro": 0.7222222222222222, + "precision_micro": 0.4444444444444444, + "precision_weighted": 0.691358024691358, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.4444444444444444, + "recall_weighted": 0.4444444444444444, + "f1": 1.0, + "f1_macro": 0.6428571428571428, + "f1_micro": 0.4444444444444444, + "f1_weighted": 0.6031746031746031 + }, + "duration": 5.376826286315918, + "num_run": 16, + "train_loss": { + "accuracy": 0.45375722543352603, + "balanced_accuracy": 0.5, + "roc_auc": 0.5, + "average_precision": 0.546242774566474, + "log_loss": 15.67221934809161, + "precision": 1.0, + "precision_macro": 0.726878612716763, + "precision_micro": 0.45375722543352603, + "precision_weighted": 0.7016188312339203, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.45375722543352603, + "recall_weighted": 0.45375722543352603, + "f1": 1.0, + "f1_macro": 0.6467289719626168, + "f1_micro": 0.45375722543352603, + "f1_weighted": 0.6140565069418183 + }, + "configuration_origin": "Random Search" + } + ] + ], + [ + [ + 1, + "{\"task_id\": \"Australian\"}", + 0, + 16.666666666666664 + ], + [ + 0.15204678362573099, + 11.514830589294434, + { + "__enum__": "StatusType.SUCCESS" + }, + 1637342750.0053334, + 1637342762.5487585, + { + "opt_loss": { + "accuracy": 0.15204678362573099, + "balanced_accuracy": 0.15263157894736845, + "roc_auc": 0.08981994459833786, + "average_precision": 0.1040861796433199, + "log_loss": 0.5765479137672738, + "precision": 0.17948717948717952, + "precision_macro": 0.15425971877584788, + "precision_micro": 0.15204678362573099, + "precision_weighted": 0.15145666758569976, + "recall": 0.1578947368421053, + "recall_macro": 0.15263157894736845, + "recall_micro": 0.15204678362573099, + "recall_weighted": 0.15204678362573099, + "f1": 0.16883116883116878, + "f1_macro": 0.15356452058579717, + "f1_micro": 0.15204678362573099, + "f1_weighted": 0.15186822633631147 + }, + "duration": 11.44463586807251, + "num_run": 8, + "train_loss": { + "accuracy": 0.09537572254335258, + "balanced_accuracy": 0.10239948774980623, + "roc_auc": 0.03963198867657458, + "average_precision": 0.044469547423341305, + "log_loss": 0.28008669264774966, + "precision": 0.03731343283582089, + "precision_macro": 0.08469445226696704, + "precision_micro": 0.09537572254335258, + "precision_weighted": 0.0890765118675354, + "recall": 0.17834394904458595, + "recall_macro": 0.10239948774980623, + "recall_micro": 0.09537572254335258, + "recall_weighted": 0.09537572254335258, + "f1": 0.11340206185567014, + "f1_macro": 0.09784816309741107, + "f1_micro": 0.09537572254335258, + "f1_weighted": 0.0964096522295953 + }, + "configuration_origin": "Default" + } + ] + ], + [ + [ + 1, + "{\"task_id\": \"Australian\"}", + 0, + 50.0 + ], + [ + 0.15204678362573099, + 15.370736837387085, + { + "__enum__": "StatusType.SUCCESS" + }, + 1637342762.794756, + 1637342779.192385, + { + "opt_loss": { + "accuracy": 0.15204678362573099, + "balanced_accuracy": 0.15263157894736845, + "roc_auc": 0.08981994459833786, + "average_precision": 0.1040861796433199, + "log_loss": 0.5765479137672738, + "precision": 0.17948717948717952, + "precision_macro": 0.15425971877584788, + "precision_micro": 0.15204678362573099, + "precision_weighted": 0.15145666758569976, + "recall": 0.1578947368421053, + "recall_macro": 0.15263157894736845, + "recall_micro": 0.15204678362573099, + "recall_weighted": 0.15204678362573099, + "f1": 0.16883116883116878, + "f1_macro": 0.15356452058579717, + "f1_micro": 0.15204678362573099, + "f1_weighted": 0.15186822633631147 + }, + "duration": 15.300711154937744, + "num_run": 8, + "train_loss": { + "accuracy": 0.09537572254335258, + "balanced_accuracy": 0.10239948774980623, + "roc_auc": 0.03963198867657458, + "average_precision": 0.044469547423341305, + "log_loss": 0.28008669264774966, + "precision": 0.03731343283582089, + "precision_macro": 0.08469445226696704, + "precision_micro": 0.09537572254335258, + "precision_weighted": 0.0890765118675354, + "recall": 0.17834394904458595, + "recall_macro": 0.10239948774980623, + "recall_micro": 0.09537572254335258, + "recall_weighted": 0.09537572254335258, + "f1": 0.11340206185567014, + "f1_macro": 0.09784816309741107, + "f1_micro": 0.09537572254335258, + "f1_weighted": 0.0964096522295953 + }, + "configuration_origin": "Default" + } + ] + ], + [ + [ + 10, + "{\"task_id\": \"Australian\"}", + 0, + 16.666666666666664 + ], + [ + 0.4035087719298246, + 23.846530199050903, + { + "__enum__": "StatusType.SUCCESS" + }, + 1637342779.4572933, + 1637342804.3368232, + { + "opt_loss": { + "accuracy": 0.4035087719298246, + "balanced_accuracy": 0.39473684210526316, + "roc_auc": 0.3946675900277007, + "average_precision": 0.4846825737029168, + "log_loss": 6.419999084913276, + "precision": 0.4639175257731959, + "precision_macro": 0.39412092504876006, + "precision_micro": 0.4035087719298246, + "precision_weighted": 0.38636574719048944, + "recall": 0.3157894736842105, + "recall_macro": 0.39473684210526316, + "recall_micro": 0.4035087719298246, + "recall_weighted": 0.4035087719298246, + "f1": 0.3988439306358381, + "f1_macro": 0.4035639771522386, + "f1_micro": 0.4035087719298246, + "f1_weighted": 0.404088426765172 + }, + "duration": 23.588075160980225, + "num_run": 17, + "train_loss": { + "accuracy": 0.3988439306358381, + "balanced_accuracy": 0.3947359552455094, + "roc_auc": 0.4153776160145586, + "average_precision": 0.49525391358194226, + "log_loss": 7.800554750687936, + "precision": 0.4486486486486486, + "precision_macro": 0.39513177774047337, + "precision_micro": 0.3988439306358381, + "precision_weighted": 0.39018224054665374, + "recall": 0.35031847133757965, + "recall_macro": 0.3947359552455094, + "recall_micro": 0.3988439306358381, + "recall_weighted": 0.3988439306358381, + "f1": 0.4035087719298246, + "f1_macro": 0.39889724310776953, + "f1_micro": 0.3988439306358381, + "f1_weighted": 0.39847074333231924 + }, + "configuration_origin": "Random Search" + } + ] + ], + [ + [ + 11, + "{\"task_id\": \"Australian\"}", + 0, + 16.666666666666664 + ], + [ + 0.4444444444444444, + 6.757539510726929, + { + "__enum__": "StatusType.SUCCESS" + }, + 1637342813.02129, + 1637342820.8067145, + { + "opt_loss": { + "accuracy": 0.4444444444444444, + "balanced_accuracy": 0.5, + "roc_auc": 0.6250692520775624, + "average_precision": 0.642743659212315, + "log_loss": 0.6874508627674036, + "precision": 1.0, + "precision_macro": 0.7222222222222222, + "precision_micro": 0.4444444444444444, + "precision_weighted": 0.691358024691358, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.4444444444444444, + "recall_weighted": 0.4444444444444444, + "f1": 1.0, + "f1_macro": 0.6428571428571428, + "f1_micro": 0.4444444444444444, + "f1_weighted": 0.6031746031746031 + }, + "duration": 6.695321321487427, + "num_run": 18, + "train_loss": { + "accuracy": 0.45375722543352603, + "balanced_accuracy": 0.5, + "roc_auc": 0.5560273649445624, + "average_precision": 0.5899410773859022, + "log_loss": 0.689653262926664, + "precision": 1.0, + "precision_macro": 0.726878612716763, + "precision_micro": 0.45375722543352603, + "precision_weighted": 0.7016188312339203, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.45375722543352603, + "recall_weighted": 0.45375722543352603, + "f1": 1.0, + "f1_macro": 0.6467289719626168, + "f1_micro": 0.45375722543352603, + "f1_weighted": 0.6140565069418183 + }, + "configuration_origin": "Random Search (sorted)" + } + ] + ], + [ + [ + 12, + "{\"task_id\": \"Australian\"}", + 0, + 16.666666666666664 + ], + [ + 0.4444444444444444, + 15.061991930007935, + { + "__enum__": "StatusType.SUCCESS" + }, + 1637342829.9214745, + 1637342846.0210106, + { + "opt_loss": { + "accuracy": 0.4444444444444444, + "balanced_accuracy": 0.5, + "roc_auc": 0.378393351800554, + "average_precision": 0.4680399341300143, + "log_loss": 0.6910723817278768, + "precision": 1.0, + "precision_macro": 0.7222222222222222, + "precision_micro": 0.4444444444444444, + "precision_weighted": 0.691358024691358, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.4444444444444444, + "recall_weighted": 0.4444444444444444, + "f1": 1.0, + "f1_macro": 0.6428571428571428, + "f1_micro": 0.4444444444444444, + "f1_weighted": 0.6031746031746031 + }, + "duration": 14.850486516952515, + "num_run": 19, + "train_loss": { + "accuracy": 0.45375722543352603, + "balanced_accuracy": 0.5, + "roc_auc": 0.44353452633707413, + "average_precision": 0.4796876232765652, + "log_loss": 0.6915661034556483, + "precision": 1.0, + "precision_macro": 0.726878612716763, + "precision_micro": 0.45375722543352603, + "precision_weighted": 0.7016188312339203, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.45375722543352603, + "recall_weighted": 0.45375722543352603, + "f1": 1.0, + "f1_macro": 0.6467289719626168, + "f1_micro": 0.45375722543352603, + "f1_weighted": 0.6140565069418183 + }, + "configuration_origin": "Random Search (sorted)" + } + ] + ], + [ + [ + 10, + "{\"task_id\": \"Australian\"}", + 0, + 50.0 + ], + [ + 1.0, + 50.010520696640015, + { + "__enum__": "StatusType.TIMEOUT" + }, + 1637342846.0745292, + 1637342897.1205413, + { + "error": "Timeout", + "configuration_origin": "Random Search" + } + ] + ], + [ + [ + 13, + "{\"task_id\": \"Australian\"}", + 0, + 50.0 + ], + [ + 1.0, + 22.011935234069824, + { + "__enum__": "StatusType.TIMEOUT" + }, + 1637342905.7068844, + 1637342928.7456856, + { + "error": "Timeout", + "configuration_origin": "Random Search (sorted)" + } + ] + ], + [ + [ + 14, + "{\"task_id\": \"Australian\"}", + 0, + 50.0 + ], + [ + 1.0, + 0.0, + { + "__enum__": "StatusType.STOP" + }, + 1637342928.8133125, + 1637342928.8133128, + {} + ] + ] + ], + "config_origins": { + "1": "Default", + "2": "Random Search", + "3": "Random Search (sorted)", + "4": "Random Search", + "5": "Random Search", + "6": "Random Search", + "7": "Random Search (sorted)", + "8": "Random Search (sorted)", + "9": "Random Search", + "10": "Random Search", + "11": "Random Search (sorted)", + "12": "Random Search (sorted)", + "13": "Random Search (sorted)", + "14": "Random Search" + }, + "configs": { + "1": { + "data_loader:batch_size": 64, + "encoder:__choice__": "OneHotEncoder", + "coalescer:__choice__": "NoCoalescer", + "feature_preprocessor:__choice__": "NoFeaturePreprocessor", + "imputer:categorical_strategy": "most_frequent", + "imputer:numerical_strategy": "mean", + "lr_scheduler:__choice__": "ReduceLROnPlateau", + "network_backbone:__choice__": "ShapedMLPBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "XavierInit", + "optimizer:__choice__": "AdamOptimizer", + "scaler:__choice__": "StandardScaler", + "trainer:__choice__": "StandardTrainer", + "lr_scheduler:ReduceLROnPlateau:factor": 0.1, + "lr_scheduler:ReduceLROnPlateau:mode": "min", + "lr_scheduler:ReduceLROnPlateau:patience": 10, + "network_backbone:ShapedMLPBackbone:activation": "relu", + "network_backbone:ShapedMLPBackbone:max_units": 200, + "network_backbone:ShapedMLPBackbone:mlp_shape": "funnel", + "network_backbone:ShapedMLPBackbone:num_groups": 5, + "network_backbone:ShapedMLPBackbone:output_dim": 200, + "network_backbone:ShapedMLPBackbone:use_dropout": false, + "network_head:fully_connected:num_layers": 2, + "network_init:XavierInit:bias_strategy": "Normal", + "optimizer:AdamOptimizer:beta1": 0.9, + "optimizer:AdamOptimizer:beta2": 0.9, + "optimizer:AdamOptimizer:lr": 0.01, + "optimizer:AdamOptimizer:weight_decay": 0.0, + "trainer:StandardTrainer:weighted_loss": true, + "network_head:fully_connected:activation": "relu", + "network_head:fully_connected:units_layer_1": 128 + }, + "2": { + "data_loader:batch_size": 142, + "encoder:__choice__": "NoEncoder", + "coalescer:__choice__": "NoCoalescer", + "feature_preprocessor:__choice__": "PowerTransformer", + "imputer:categorical_strategy": "constant_!missing!", + "imputer:numerical_strategy": "median", + "lr_scheduler:__choice__": "NoScheduler", + "network_backbone:__choice__": "ShapedResNetBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "KaimingInit", + "optimizer:__choice__": "AdamOptimizer", + "scaler:__choice__": "Normalizer", + "trainer:__choice__": "MixUpTrainer", + "feature_preprocessor:PowerTransformer:standardize": true, + "network_backbone:ShapedResNetBackbone:activation": "relu", + "network_backbone:ShapedResNetBackbone:blocks_per_group": 1, + "network_backbone:ShapedResNetBackbone:max_units": 175, + "network_backbone:ShapedResNetBackbone:num_groups": 3, + "network_backbone:ShapedResNetBackbone:output_dim": 550, + "network_backbone:ShapedResNetBackbone:resnet_shape": "funnel", + "network_backbone:ShapedResNetBackbone:use_dropout": false, + "network_backbone:ShapedResNetBackbone:use_shake_drop": true, + "network_backbone:ShapedResNetBackbone:use_shake_shake": true, + "network_head:fully_connected:num_layers": 2, + "network_init:KaimingInit:bias_strategy": "Normal", + "optimizer:AdamOptimizer:beta1": 0.8660298289969375, + "optimizer:AdamOptimizer:beta2": 0.9517157453274235, + "optimizer:AdamOptimizer:lr": 1.0377748473731365e-05, + "optimizer:AdamOptimizer:weight_decay": 0.07437634123996516, + "scaler:Normalizer:norm": "mean_abs", + "trainer:MixUpTrainer:alpha": 0.13179357367568267, + "trainer:MixUpTrainer:weighted_loss": true, + "network_backbone:ShapedResNetBackbone:max_shake_drop_probability": 0.7993610769045779, + "network_head:fully_connected:activation": "sigmoid", + "network_head:fully_connected:units_layer_1": 308 + }, + "3": { + "data_loader:batch_size": 246, + "encoder:__choice__": "OneHotEncoder", + "coalescer:__choice__": "NoCoalescer", + "feature_preprocessor:__choice__": "PowerTransformer", + "imputer:categorical_strategy": "constant_!missing!", + "imputer:numerical_strategy": "most_frequent", + "lr_scheduler:__choice__": "CosineAnnealingWarmRestarts", + "network_backbone:__choice__": "ResNetBackbone", + "network_embedding:__choice__": "LearnedEntityEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "XavierInit", + "optimizer:__choice__": "AdamOptimizer", + "scaler:__choice__": "Normalizer", + "trainer:__choice__": "StandardTrainer", + "feature_preprocessor:PowerTransformer:standardize": true, + "lr_scheduler:CosineAnnealingWarmRestarts:T_0": 17, + "lr_scheduler:CosineAnnealingWarmRestarts:T_mult": 1.0577034671447638, + "network_backbone:ResNetBackbone:activation": "sigmoid", + "network_backbone:ResNetBackbone:blocks_per_group_0": 1, + "network_backbone:ResNetBackbone:blocks_per_group_1": 4, + "network_backbone:ResNetBackbone:num_groups": 10, + "network_backbone:ResNetBackbone:num_units_0": 974, + "network_backbone:ResNetBackbone:num_units_1": 151, + "network_backbone:ResNetBackbone:use_dropout": true, + "network_backbone:ResNetBackbone:use_shake_drop": true, + "network_backbone:ResNetBackbone:use_shake_shake": true, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_0": 0.6898659803969109, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_1": 0.6193894885012183, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_2": 0.27044405840757246, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_3": 0.3353276257116905, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_4": 0.25330009522745545, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_5": 0.28087428370045076, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_6": 0.985667346693578, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_7": 0.532995443030165, + "network_embedding:LearnedEntityEmbedding:min_unique_values_for_embedding": 4, + "network_head:fully_connected:num_layers": 4, + "network_init:XavierInit:bias_strategy": "Normal", + "optimizer:AdamOptimizer:beta1": 0.9319239463981089, + "optimizer:AdamOptimizer:beta2": 0.9290660642046109, + "optimizer:AdamOptimizer:lr": 4.934398361769327e-05, + "optimizer:AdamOptimizer:weight_decay": 0.0647885374302594, + "scaler:Normalizer:norm": "mean_abs", + "trainer:StandardTrainer:weighted_loss": true, + "network_backbone:ResNetBackbone:blocks_per_group_10": 2, + "network_backbone:ResNetBackbone:blocks_per_group_2": 1, + "network_backbone:ResNetBackbone:blocks_per_group_3": 1, + "network_backbone:ResNetBackbone:blocks_per_group_4": 3, + "network_backbone:ResNetBackbone:blocks_per_group_5": 1, + "network_backbone:ResNetBackbone:blocks_per_group_6": 3, + "network_backbone:ResNetBackbone:blocks_per_group_7": 4, + "network_backbone:ResNetBackbone:blocks_per_group_8": 1, + "network_backbone:ResNetBackbone:blocks_per_group_9": 3, + "network_backbone:ResNetBackbone:dropout_0": 0.1998483800982469, + "network_backbone:ResNetBackbone:dropout_1": 0.21671729531007777, + "network_backbone:ResNetBackbone:dropout_10": 0.2027668457966562, + "network_backbone:ResNetBackbone:dropout_2": 0.7140248388727144, + "network_backbone:ResNetBackbone:dropout_3": 0.1324478677866992, + "network_backbone:ResNetBackbone:dropout_4": 0.26711076053122573, + "network_backbone:ResNetBackbone:dropout_5": 0.2895993889716623, + "network_backbone:ResNetBackbone:dropout_6": 0.047419135928320616, + "network_backbone:ResNetBackbone:dropout_7": 0.593522761474697, + "network_backbone:ResNetBackbone:dropout_8": 0.11825542268484464, + "network_backbone:ResNetBackbone:dropout_9": 0.5802180655508312, + "network_backbone:ResNetBackbone:max_shake_drop_probability": 0.8422119101175598, + "network_backbone:ResNetBackbone:num_units_10": 1012, + "network_backbone:ResNetBackbone:num_units_2": 793, + "network_backbone:ResNetBackbone:num_units_3": 184, + "network_backbone:ResNetBackbone:num_units_4": 1022, + "network_backbone:ResNetBackbone:num_units_5": 88, + "network_backbone:ResNetBackbone:num_units_6": 666, + "network_backbone:ResNetBackbone:num_units_7": 927, + "network_backbone:ResNetBackbone:num_units_8": 614, + "network_backbone:ResNetBackbone:num_units_9": 552, + "network_head:fully_connected:activation": "relu", + "network_head:fully_connected:units_layer_1": 92, + "network_head:fully_connected:units_layer_2": 202, + "network_head:fully_connected:units_layer_3": 171 + }, + "4": { + "data_loader:batch_size": 269, + "encoder:__choice__": "OneHotEncoder", + "coalescer:__choice__": "NoCoalescer", + "feature_preprocessor:__choice__": "PowerTransformer", + "imputer:categorical_strategy": "constant_!missing!", + "imputer:numerical_strategy": "median", + "lr_scheduler:__choice__": "CosineAnnealingLR", + "network_backbone:__choice__": "ShapedMLPBackbone", + "network_embedding:__choice__": "LearnedEntityEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "KaimingInit", + "optimizer:__choice__": "RMSpropOptimizer", + "scaler:__choice__": "MinMaxScaler", + "trainer:__choice__": "StandardTrainer", + "feature_preprocessor:PowerTransformer:standardize": true, + "lr_scheduler:CosineAnnealingLR:T_max": 57, + "network_backbone:ShapedMLPBackbone:activation": "relu", + "network_backbone:ShapedMLPBackbone:max_units": 199, + "network_backbone:ShapedMLPBackbone:mlp_shape": "stairs", + "network_backbone:ShapedMLPBackbone:num_groups": 12, + "network_backbone:ShapedMLPBackbone:output_dim": 641, + "network_backbone:ShapedMLPBackbone:use_dropout": true, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_0": 0.8093046015402414, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_1": 0.6952888698136637, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_2": 0.7136167420874352, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_3": 0.7071870846094686, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_4": 0.8821351885181623, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_5": 0.21840740866837938, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_6": 0.7366390825638998, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_7": 0.548715467945816, + "network_embedding:LearnedEntityEmbedding:min_unique_values_for_embedding": 7, + "network_head:fully_connected:num_layers": 3, + "network_init:KaimingInit:bias_strategy": "Zero", + "optimizer:RMSpropOptimizer:alpha": 0.23716801972855298, + "optimizer:RMSpropOptimizer:lr": 0.0011708542709120838, + "optimizer:RMSpropOptimizer:momentum": 0.5620565618493047, + "optimizer:RMSpropOptimizer:weight_decay": 0.05858239202799009, + "trainer:StandardTrainer:weighted_loss": true, + "network_backbone:ShapedMLPBackbone:max_dropout": 0.20819857031346878, + "network_head:fully_connected:activation": "relu", + "network_head:fully_connected:units_layer_1": 177, + "network_head:fully_connected:units_layer_2": 196 + }, + "5": { + "data_loader:batch_size": 191, + "encoder:__choice__": "OneHotEncoder", + "coalescer:__choice__": "NoCoalescer", + "feature_preprocessor:__choice__": "RandomKitchenSinks", + "imputer:categorical_strategy": "constant_!missing!", + "imputer:numerical_strategy": "most_frequent", + "lr_scheduler:__choice__": "ExponentialLR", + "network_backbone:__choice__": "ShapedResNetBackbone", + "network_embedding:__choice__": "LearnedEntityEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "SparseInit", + "optimizer:__choice__": "AdamOptimizer", + "scaler:__choice__": "MinMaxScaler", + "trainer:__choice__": "MixUpTrainer", + "feature_preprocessor:RandomKitchenSinks:gamma": 0.00023806069646323692, + "feature_preprocessor:RandomKitchenSinks:n_components": 6, + "lr_scheduler:ExponentialLR:gamma": 0.7718494018636944, + "network_backbone:ShapedResNetBackbone:activation": "tanh", + "network_backbone:ShapedResNetBackbone:blocks_per_group": 3, + "network_backbone:ShapedResNetBackbone:max_units": 869, + "network_backbone:ShapedResNetBackbone:num_groups": 2, + "network_backbone:ShapedResNetBackbone:output_dim": 868, + "network_backbone:ShapedResNetBackbone:resnet_shape": "triangle", + "network_backbone:ShapedResNetBackbone:use_dropout": true, + "network_backbone:ShapedResNetBackbone:use_shake_drop": true, + "network_backbone:ShapedResNetBackbone:use_shake_shake": true, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_0": 0.08846693746970624, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_1": 0.6597252449477167, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_2": 0.11290616066859738, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_3": 0.4187266624427779, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_4": 0.026810815995375492, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_5": 0.02021466731982824, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_6": 0.01616376260397212, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_7": 0.6463510235731745, + "network_embedding:LearnedEntityEmbedding:min_unique_values_for_embedding": 6, + "network_head:fully_connected:num_layers": 3, + "network_init:SparseInit:bias_strategy": "Normal", + "optimizer:AdamOptimizer:beta1": 0.9966044931531224, + "optimizer:AdamOptimizer:beta2": 0.9293356180290759, + "optimizer:AdamOptimizer:lr": 0.07180366191531826, + "optimizer:AdamOptimizer:weight_decay": 0.012304534471441598, + "trainer:MixUpTrainer:alpha": 0.8900376828213522, + "trainer:MixUpTrainer:weighted_loss": true, + "network_backbone:ShapedResNetBackbone:max_dropout": 0.6688622458251051, + "network_backbone:ShapedResNetBackbone:max_shake_drop_probability": 0.28903761225065516, + "network_head:fully_connected:activation": "sigmoid", + "network_head:fully_connected:units_layer_1": 198, + "network_head:fully_connected:units_layer_2": 283 + }, + "6": { + "data_loader:batch_size": 53, + "encoder:__choice__": "OneHotEncoder", + "coalescer:__choice__": "NoCoalescer", + "feature_preprocessor:__choice__": "PowerTransformer", + "imputer:categorical_strategy": "constant_!missing!", + "imputer:numerical_strategy": "median", + "lr_scheduler:__choice__": "StepLR", + "network_backbone:__choice__": "ResNetBackbone", + "network_embedding:__choice__": "LearnedEntityEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "OrthogonalInit", + "optimizer:__choice__": "AdamOptimizer", + "scaler:__choice__": "Normalizer", + "trainer:__choice__": "MixUpTrainer", + "feature_preprocessor:PowerTransformer:standardize": false, + "lr_scheduler:StepLR:gamma": 0.27529217359012764, + "lr_scheduler:StepLR:step_size": 8, + "network_backbone:ResNetBackbone:activation": "sigmoid", + "network_backbone:ResNetBackbone:blocks_per_group_0": 3, + "network_backbone:ResNetBackbone:blocks_per_group_1": 1, + "network_backbone:ResNetBackbone:num_groups": 6, + "network_backbone:ResNetBackbone:num_units_0": 884, + "network_backbone:ResNetBackbone:num_units_1": 160, + "network_backbone:ResNetBackbone:use_dropout": false, + "network_backbone:ResNetBackbone:use_shake_drop": true, + "network_backbone:ResNetBackbone:use_shake_shake": false, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_0": 0.5659324295712268, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_1": 0.8744001957677244, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_2": 0.3415903412295024, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_3": 0.8599314829187148, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_4": 0.9869678384973877, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_5": 0.7490528427155283, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_6": 0.9979477892240094, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_7": 0.4171316119626819, + "network_embedding:LearnedEntityEmbedding:min_unique_values_for_embedding": 5, + "network_head:fully_connected:num_layers": 1, + "network_init:OrthogonalInit:bias_strategy": "Zero", + "optimizer:AdamOptimizer:beta1": 0.9576776568384536, + "optimizer:AdamOptimizer:beta2": 0.9605074039230137, + "optimizer:AdamOptimizer:lr": 0.02098507521065345, + "optimizer:AdamOptimizer:weight_decay": 0.021686007599294888, + "scaler:Normalizer:norm": "mean_abs", + "trainer:MixUpTrainer:alpha": 0.8399712211486785, + "trainer:MixUpTrainer:weighted_loss": true, + "network_backbone:ResNetBackbone:blocks_per_group_2": 3, + "network_backbone:ResNetBackbone:blocks_per_group_3": 3, + "network_backbone:ResNetBackbone:blocks_per_group_4": 1, + "network_backbone:ResNetBackbone:blocks_per_group_5": 1, + "network_backbone:ResNetBackbone:blocks_per_group_6": 3, + "network_backbone:ResNetBackbone:max_shake_drop_probability": 0.09160627667494659, + "network_backbone:ResNetBackbone:num_units_2": 396, + "network_backbone:ResNetBackbone:num_units_3": 587, + "network_backbone:ResNetBackbone:num_units_4": 169, + "network_backbone:ResNetBackbone:num_units_5": 546, + "network_backbone:ResNetBackbone:num_units_6": 92 + }, + "7": { + "data_loader:batch_size": 232, + "encoder:__choice__": "NoEncoder", + "coalescer:__choice__": "NoCoalescer", + "feature_preprocessor:__choice__": "RandomKitchenSinks", + "imputer:categorical_strategy": "most_frequent", + "imputer:numerical_strategy": "most_frequent", + "lr_scheduler:__choice__": "NoScheduler", + "network_backbone:__choice__": "ResNetBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "XavierInit", + "optimizer:__choice__": "AdamOptimizer", + "scaler:__choice__": "Normalizer", + "trainer:__choice__": "StandardTrainer", + "feature_preprocessor:RandomKitchenSinks:gamma": 5.7968061542283495e-05, + "feature_preprocessor:RandomKitchenSinks:n_components": 5, + "network_backbone:ResNetBackbone:activation": "sigmoid", + "network_backbone:ResNetBackbone:blocks_per_group_0": 2, + "network_backbone:ResNetBackbone:blocks_per_group_1": 3, + "network_backbone:ResNetBackbone:num_groups": 14, + "network_backbone:ResNetBackbone:num_units_0": 63, + "network_backbone:ResNetBackbone:num_units_1": 229, + "network_backbone:ResNetBackbone:use_dropout": true, + "network_backbone:ResNetBackbone:use_shake_drop": false, + "network_backbone:ResNetBackbone:use_shake_shake": true, + "network_head:fully_connected:num_layers": 2, + "network_init:XavierInit:bias_strategy": "Normal", + "optimizer:AdamOptimizer:beta1": 0.9646917651093316, + "optimizer:AdamOptimizer:beta2": 0.9949552394978046, + "optimizer:AdamOptimizer:lr": 0.018422761006289576, + "optimizer:AdamOptimizer:weight_decay": 0.01700341747601285, + "scaler:Normalizer:norm": "mean_squared", + "trainer:StandardTrainer:weighted_loss": false, + "network_backbone:ResNetBackbone:blocks_per_group_10": 4, + "network_backbone:ResNetBackbone:blocks_per_group_11": 3, + "network_backbone:ResNetBackbone:blocks_per_group_12": 2, + "network_backbone:ResNetBackbone:blocks_per_group_13": 1, + "network_backbone:ResNetBackbone:blocks_per_group_14": 3, + "network_backbone:ResNetBackbone:blocks_per_group_2": 1, + "network_backbone:ResNetBackbone:blocks_per_group_3": 3, + "network_backbone:ResNetBackbone:blocks_per_group_4": 4, + "network_backbone:ResNetBackbone:blocks_per_group_5": 3, + "network_backbone:ResNetBackbone:blocks_per_group_6": 4, + "network_backbone:ResNetBackbone:blocks_per_group_7": 2, + "network_backbone:ResNetBackbone:blocks_per_group_8": 3, + "network_backbone:ResNetBackbone:blocks_per_group_9": 2, + "network_backbone:ResNetBackbone:dropout_0": 0.3872694110962167, + "network_backbone:ResNetBackbone:dropout_1": 0.7182095865182352, + "network_backbone:ResNetBackbone:dropout_10": 0.7518775284870586, + "network_backbone:ResNetBackbone:dropout_11": 0.3717581189860213, + "network_backbone:ResNetBackbone:dropout_12": 0.055178370982331075, + "network_backbone:ResNetBackbone:dropout_13": 0.5670307132839905, + "network_backbone:ResNetBackbone:dropout_14": 0.7859566818562779, + "network_backbone:ResNetBackbone:dropout_2": 0.5796670187291707, + "network_backbone:ResNetBackbone:dropout_3": 0.05370643307213783, + "network_backbone:ResNetBackbone:dropout_4": 0.37288408223729974, + "network_backbone:ResNetBackbone:dropout_5": 0.47179695650262793, + "network_backbone:ResNetBackbone:dropout_6": 0.20070003914010803, + "network_backbone:ResNetBackbone:dropout_7": 0.638048407623313, + "network_backbone:ResNetBackbone:dropout_8": 0.6190670404279601, + "network_backbone:ResNetBackbone:dropout_9": 0.33325853682297146, + "network_backbone:ResNetBackbone:num_units_10": 925, + "network_backbone:ResNetBackbone:num_units_11": 164, + "network_backbone:ResNetBackbone:num_units_12": 247, + "network_backbone:ResNetBackbone:num_units_13": 339, + "network_backbone:ResNetBackbone:num_units_14": 769, + "network_backbone:ResNetBackbone:num_units_2": 502, + "network_backbone:ResNetBackbone:num_units_3": 101, + "network_backbone:ResNetBackbone:num_units_4": 842, + "network_backbone:ResNetBackbone:num_units_5": 906, + "network_backbone:ResNetBackbone:num_units_6": 933, + "network_backbone:ResNetBackbone:num_units_7": 329, + "network_backbone:ResNetBackbone:num_units_8": 898, + "network_backbone:ResNetBackbone:num_units_9": 161, + "network_head:fully_connected:activation": "relu", + "network_head:fully_connected:units_layer_1": 327 + }, + "8": { + "data_loader:batch_size": 164, + "encoder:__choice__": "NoEncoder", + "coalescer:__choice__": "NoCoalescer", + "feature_preprocessor:__choice__": "NoFeaturePreprocessor", + "imputer:categorical_strategy": "most_frequent", + "imputer:numerical_strategy": "mean", + "lr_scheduler:__choice__": "StepLR", + "network_backbone:__choice__": "ShapedMLPBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "NoInit", + "optimizer:__choice__": "RMSpropOptimizer", + "scaler:__choice__": "MinMaxScaler", + "trainer:__choice__": "MixUpTrainer", + "lr_scheduler:StepLR:gamma": 0.2905213739360219, + "lr_scheduler:StepLR:step_size": 10, + "network_backbone:ShapedMLPBackbone:activation": "sigmoid", + "network_backbone:ShapedMLPBackbone:max_units": 903, + "network_backbone:ShapedMLPBackbone:mlp_shape": "brick", + "network_backbone:ShapedMLPBackbone:num_groups": 10, + "network_backbone:ShapedMLPBackbone:output_dim": 943, + "network_backbone:ShapedMLPBackbone:use_dropout": false, + "network_head:fully_connected:num_layers": 3, + "network_init:NoInit:bias_strategy": "Zero", + "optimizer:RMSpropOptimizer:alpha": 0.25445785033325663, + "optimizer:RMSpropOptimizer:lr": 0.00012058949092384073, + "optimizer:RMSpropOptimizer:momentum": 0.6601732030357997, + "optimizer:RMSpropOptimizer:weight_decay": 0.030275825765581223, + "trainer:MixUpTrainer:alpha": 0.2222082093355312, + "trainer:MixUpTrainer:weighted_loss": true, + "network_head:fully_connected:activation": "sigmoid", + "network_head:fully_connected:units_layer_1": 110, + "network_head:fully_connected:units_layer_2": 70 + }, + "9": { + "data_loader:batch_size": 94, + "encoder:__choice__": "NoEncoder", + "coalescer:__choice__": "NoCoalescer", + "feature_preprocessor:__choice__": "PolynomialFeatures", + "imputer:categorical_strategy": "most_frequent", + "imputer:numerical_strategy": "mean", + "lr_scheduler:__choice__": "CyclicLR", + "network_backbone:__choice__": "MLPBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "KaimingInit", + "optimizer:__choice__": "RMSpropOptimizer", + "scaler:__choice__": "StandardScaler", + "trainer:__choice__": "StandardTrainer", + "feature_preprocessor:PolynomialFeatures:degree": 2, + "feature_preprocessor:PolynomialFeatures:include_bias": false, + "feature_preprocessor:PolynomialFeatures:interaction_only": false, + "lr_scheduler:CyclicLR:base_lr": 0.09510483864725039, + "lr_scheduler:CyclicLR:max_lr": 0.026215723559513626, + "lr_scheduler:CyclicLR:mode": "triangular", + "lr_scheduler:CyclicLR:step_size_up": 3829, + "network_backbone:MLPBackbone:activation": "sigmoid", + "network_backbone:MLPBackbone:num_groups": 7, + "network_backbone:MLPBackbone:num_units_1": 47, + "network_backbone:MLPBackbone:use_dropout": true, + "network_head:fully_connected:num_layers": 3, + "network_init:KaimingInit:bias_strategy": "Zero", + "optimizer:RMSpropOptimizer:alpha": 0.75085811094601, + "optimizer:RMSpropOptimizer:lr": 0.0002950013672615944, + "optimizer:RMSpropOptimizer:momentum": 0.515129966307681, + "optimizer:RMSpropOptimizer:weight_decay": 0.01979731884468683, + "trainer:StandardTrainer:weighted_loss": true, + "network_backbone:MLPBackbone:dropout_1": 0.5362963908147109, + "network_backbone:MLPBackbone:dropout_2": 0.09403575191589564, + "network_backbone:MLPBackbone:dropout_3": 0.5576340928985162, + "network_backbone:MLPBackbone:dropout_4": 0.3102921398336836, + "network_backbone:MLPBackbone:dropout_5": 0.36841155269138837, + "network_backbone:MLPBackbone:dropout_6": 0.459182557172949, + "network_backbone:MLPBackbone:dropout_7": 0.2741849570242409, + "network_backbone:MLPBackbone:num_units_2": 323, + "network_backbone:MLPBackbone:num_units_3": 424, + "network_backbone:MLPBackbone:num_units_4": 637, + "network_backbone:MLPBackbone:num_units_5": 668, + "network_backbone:MLPBackbone:num_units_6": 507, + "network_backbone:MLPBackbone:num_units_7": 972, + "network_head:fully_connected:activation": "sigmoid", + "network_head:fully_connected:units_layer_1": 482, + "network_head:fully_connected:units_layer_2": 425 + }, + "10": { + "data_loader:batch_size": 70, + "encoder:__choice__": "OneHotEncoder", + "coalescer:__choice__": "NoCoalescer", + "feature_preprocessor:__choice__": "PowerTransformer", + "imputer:categorical_strategy": "most_frequent", + "imputer:numerical_strategy": "constant_zero", + "lr_scheduler:__choice__": "CyclicLR", + "network_backbone:__choice__": "ShapedMLPBackbone", + "network_embedding:__choice__": "LearnedEntityEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "SparseInit", + "optimizer:__choice__": "RMSpropOptimizer", + "scaler:__choice__": "NoScaler", + "trainer:__choice__": "MixUpTrainer", + "feature_preprocessor:PowerTransformer:standardize": false, + "lr_scheduler:CyclicLR:base_lr": 0.05611929109855669, + "lr_scheduler:CyclicLR:max_lr": 0.01831055731936772, + "lr_scheduler:CyclicLR:mode": "triangular2", + "lr_scheduler:CyclicLR:step_size_up": 3104, + "network_backbone:ShapedMLPBackbone:activation": "tanh", + "network_backbone:ShapedMLPBackbone:max_units": 759, + "network_backbone:ShapedMLPBackbone:mlp_shape": "brick", + "network_backbone:ShapedMLPBackbone:num_groups": 10, + "network_backbone:ShapedMLPBackbone:output_dim": 155, + "network_backbone:ShapedMLPBackbone:use_dropout": true, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_0": 0.14333490052026576, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_1": 0.7794060644969828, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_2": 0.28020395999441905, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_3": 0.2820327943739419, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_4": 0.7390548552027222, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_5": 0.025302711343403672, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_6": 0.5677825375428477, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_7": 0.7093786601691139, + "network_embedding:LearnedEntityEmbedding:min_unique_values_for_embedding": 6, + "network_head:fully_connected:num_layers": 4, + "network_init:SparseInit:bias_strategy": "Zero", + "optimizer:RMSpropOptimizer:alpha": 0.15659144532965727, + "optimizer:RMSpropOptimizer:lr": 0.015691888676781927, + "optimizer:RMSpropOptimizer:momentum": 0.30317416976729206, + "optimizer:RMSpropOptimizer:weight_decay": 0.010642526008626797, + "trainer:MixUpTrainer:alpha": 0.3709089665342886, + "trainer:MixUpTrainer:weighted_loss": false, + "network_backbone:ShapedMLPBackbone:max_dropout": 0.3789762581174825, + "network_head:fully_connected:activation": "tanh", + "network_head:fully_connected:units_layer_1": 499, + "network_head:fully_connected:units_layer_2": 465, + "network_head:fully_connected:units_layer_3": 238 + }, + "11": { + "data_loader:batch_size": 274, + "encoder:__choice__": "NoEncoder", + "coalescer:__choice__": "NoCoalescer", + "feature_preprocessor:__choice__": "RandomKitchenSinks", + "imputer:categorical_strategy": "constant_!missing!", + "imputer:numerical_strategy": "mean", + "lr_scheduler:__choice__": "CyclicLR", + "network_backbone:__choice__": "ShapedMLPBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "SparseInit", + "optimizer:__choice__": "AdamOptimizer", + "scaler:__choice__": "Normalizer", + "trainer:__choice__": "StandardTrainer", + "feature_preprocessor:RandomKitchenSinks:gamma": 0.00017836687895829377, + "feature_preprocessor:RandomKitchenSinks:n_components": 3, + "lr_scheduler:CyclicLR:base_lr": 0.061001847805883254, + "lr_scheduler:CyclicLR:max_lr": 0.037867703829357294, + "lr_scheduler:CyclicLR:mode": "triangular", + "lr_scheduler:CyclicLR:step_size_up": 3395, + "network_backbone:ShapedMLPBackbone:activation": "relu", + "network_backbone:ShapedMLPBackbone:max_units": 94, + "network_backbone:ShapedMLPBackbone:mlp_shape": "diamond", + "network_backbone:ShapedMLPBackbone:num_groups": 4, + "network_backbone:ShapedMLPBackbone:output_dim": 763, + "network_backbone:ShapedMLPBackbone:use_dropout": false, + "network_head:fully_connected:num_layers": 3, + "network_init:SparseInit:bias_strategy": "Zero", + "optimizer:AdamOptimizer:beta1": 0.9010241766841086, + "optimizer:AdamOptimizer:beta2": 0.9275862063741073, + "optimizer:AdamOptimizer:lr": 0.00048241454070108375, + "optimizer:AdamOptimizer:weight_decay": 0.058438892093437125, + "scaler:Normalizer:norm": "max", + "trainer:StandardTrainer:weighted_loss": true, + "network_head:fully_connected:activation": "relu", + "network_head:fully_connected:units_layer_1": 293, + "network_head:fully_connected:units_layer_2": 177 + }, + "12": { + "data_loader:batch_size": 191, + "encoder:__choice__": "NoEncoder", + "coalescer:__choice__": "NoCoalescer", + "feature_preprocessor:__choice__": "NoFeaturePreprocessor", + "imputer:categorical_strategy": "constant_!missing!", + "imputer:numerical_strategy": "median", + "lr_scheduler:__choice__": "CosineAnnealingWarmRestarts", + "network_backbone:__choice__": "ResNetBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "XavierInit", + "optimizer:__choice__": "RMSpropOptimizer", + "scaler:__choice__": "Normalizer", + "trainer:__choice__": "StandardTrainer", + "lr_scheduler:CosineAnnealingWarmRestarts:T_0": 18, + "lr_scheduler:CosineAnnealingWarmRestarts:T_mult": 1.7405132785152093, + "network_backbone:ResNetBackbone:activation": "relu", + "network_backbone:ResNetBackbone:blocks_per_group_0": 1, + "network_backbone:ResNetBackbone:blocks_per_group_1": 4, + "network_backbone:ResNetBackbone:num_groups": 6, + "network_backbone:ResNetBackbone:num_units_0": 894, + "network_backbone:ResNetBackbone:num_units_1": 395, + "network_backbone:ResNetBackbone:use_dropout": true, + "network_backbone:ResNetBackbone:use_shake_drop": false, + "network_backbone:ResNetBackbone:use_shake_shake": false, + "network_head:fully_connected:num_layers": 4, + "network_init:XavierInit:bias_strategy": "Zero", + "optimizer:RMSpropOptimizer:alpha": 0.6521242194975473, + "optimizer:RMSpropOptimizer:lr": 4.097035283946373e-05, + "optimizer:RMSpropOptimizer:momentum": 0.1792833337110808, + "optimizer:RMSpropOptimizer:weight_decay": 0.006909623450893943, + "scaler:Normalizer:norm": "mean_abs", + "trainer:StandardTrainer:weighted_loss": false, + "network_backbone:ResNetBackbone:blocks_per_group_2": 2, + "network_backbone:ResNetBackbone:blocks_per_group_3": 3, + "network_backbone:ResNetBackbone:blocks_per_group_4": 2, + "network_backbone:ResNetBackbone:blocks_per_group_5": 4, + "network_backbone:ResNetBackbone:blocks_per_group_6": 1, + "network_backbone:ResNetBackbone:dropout_0": 0.6575114752945207, + "network_backbone:ResNetBackbone:dropout_1": 0.28916184819601504, + "network_backbone:ResNetBackbone:dropout_2": 0.09888388652277876, + "network_backbone:ResNetBackbone:dropout_3": 0.791809735686961, + "network_backbone:ResNetBackbone:dropout_4": 0.06432675017963892, + "network_backbone:ResNetBackbone:dropout_5": 0.3015819044494064, + "network_backbone:ResNetBackbone:dropout_6": 0.792332044450592, + "network_backbone:ResNetBackbone:num_units_2": 173, + "network_backbone:ResNetBackbone:num_units_3": 290, + "network_backbone:ResNetBackbone:num_units_4": 633, + "network_backbone:ResNetBackbone:num_units_5": 16, + "network_backbone:ResNetBackbone:num_units_6": 542, + "network_head:fully_connected:activation": "relu", + "network_head:fully_connected:units_layer_1": 429, + "network_head:fully_connected:units_layer_2": 342, + "network_head:fully_connected:units_layer_3": 322 + }, + "13": { + "data_loader:batch_size": 35, + "encoder:__choice__": "OneHotEncoder", + "coalescer:__choice__": "NoCoalescer", + "feature_preprocessor:__choice__": "PowerTransformer", + "imputer:categorical_strategy": "most_frequent", + "imputer:numerical_strategy": "most_frequent", + "lr_scheduler:__choice__": "ExponentialLR", + "network_backbone:__choice__": "ShapedMLPBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "XavierInit", + "optimizer:__choice__": "AdamWOptimizer", + "scaler:__choice__": "Normalizer", + "trainer:__choice__": "MixUpTrainer", + "feature_preprocessor:PowerTransformer:standardize": false, + "lr_scheduler:ExponentialLR:gamma": 0.863670772292724, + "network_backbone:ShapedMLPBackbone:activation": "sigmoid", + "network_backbone:ShapedMLPBackbone:max_units": 957, + "network_backbone:ShapedMLPBackbone:mlp_shape": "long_funnel", + "network_backbone:ShapedMLPBackbone:num_groups": 7, + "network_backbone:ShapedMLPBackbone:output_dim": 16, + "network_backbone:ShapedMLPBackbone:use_dropout": true, + "network_head:fully_connected:num_layers": 3, + "network_init:XavierInit:bias_strategy": "Normal", + "optimizer:AdamWOptimizer:beta1": 0.9298951109018316, + "optimizer:AdamWOptimizer:beta2": 0.9367719861032991, + "optimizer:AdamWOptimizer:lr": 2.3043911799203502e-05, + "optimizer:AdamWOptimizer:weight_decay": 0.08948752020001628, + "scaler:Normalizer:norm": "max", + "trainer:MixUpTrainer:alpha": 0.1848582510096881, + "trainer:MixUpTrainer:weighted_loss": true, + "network_backbone:ShapedMLPBackbone:max_dropout": 0.4933977554884884, + "network_head:fully_connected:activation": "relu", + "network_head:fully_connected:units_layer_1": 105, + "network_head:fully_connected:units_layer_2": 185 + }, + "14": { + "data_loader:batch_size": 154, + "encoder:__choice__": "OneHotEncoder", + "coalescer:__choice__": "NoCoalescer", + "feature_preprocessor:__choice__": "KernelPCA", + "imputer:categorical_strategy": "most_frequent", + "imputer:numerical_strategy": "mean", + "lr_scheduler:__choice__": "StepLR", + "network_backbone:__choice__": "ResNetBackbone", + "network_embedding:__choice__": "LearnedEntityEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "NoInit", + "optimizer:__choice__": "AdamWOptimizer", + "scaler:__choice__": "NoScaler", + "trainer:__choice__": "StandardTrainer", + "feature_preprocessor:KernelPCA:kernel": "sigmoid", + "feature_preprocessor:KernelPCA:n_components": 3, + "lr_scheduler:StepLR:gamma": 0.5658285105415104, + "lr_scheduler:StepLR:step_size": 10, + "network_backbone:ResNetBackbone:activation": "tanh", + "network_backbone:ResNetBackbone:blocks_per_group_0": 2, + "network_backbone:ResNetBackbone:blocks_per_group_1": 2, + "network_backbone:ResNetBackbone:num_groups": 1, + "network_backbone:ResNetBackbone:num_units_0": 623, + "network_backbone:ResNetBackbone:num_units_1": 42, + "network_backbone:ResNetBackbone:use_dropout": false, + "network_backbone:ResNetBackbone:use_shake_drop": true, + "network_backbone:ResNetBackbone:use_shake_shake": true, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_0": 0.7061800992159439, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_1": 0.40404533505032336, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_2": 0.14124612419045746, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_3": 0.24304972767199295, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_4": 0.8403938666630251, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_5": 0.11081539209354929, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_6": 0.5150164644256714, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_7": 0.6185258490472787, + "network_embedding:LearnedEntityEmbedding:min_unique_values_for_embedding": 3, + "network_head:fully_connected:num_layers": 2, + "network_init:NoInit:bias_strategy": "Zero", + "optimizer:AdamWOptimizer:beta1": 0.9639206805787317, + "optimizer:AdamWOptimizer:beta2": 0.9439342949959634, + "optimizer:AdamWOptimizer:lr": 0.05110804312778185, + "optimizer:AdamWOptimizer:weight_decay": 0.026136253949706992, + "trainer:StandardTrainer:weighted_loss": true, + "feature_preprocessor:KernelPCA:coef0": 0.27733876378393374, + "network_backbone:ResNetBackbone:max_shake_drop_probability": 0.4280891218905112, + "network_head:fully_connected:activation": "relu", + "network_head:fully_connected:units_layer_1": 506 + } + } + } \ No newline at end of file diff --git a/test/test_utils/test_results_manager.py b/test/test_utils/test_results_manager.py index 496aec7fa..102526e1e 100644 --- a/test/test_utils/test_results_manager.py +++ b/test/test_utils/test_results_manager.py @@ -404,6 +404,59 @@ def test_search_results_sprint_statistics(): assert all([m1 == m2 for m1, m2 in zip(api.sprint_statistics().split("\n"), msg)]) +def test_search_results_sprint_statistics_no_test(): + BaseTask.__abstractmethods__ = set() + api = BaseTask() + for method in ['get_search_results', 'sprint_statistics', 'get_incumbent_results']: + with pytest.raises(RuntimeError): + getattr(api, method)() + + run_history_data = json.load(open(os.path.join(os.path.dirname(__file__), + 'runhistory_no_test.json'), + mode='r'))['data'] + api._results_manager.run_history = MagicMock() + api.run_history.empty = MagicMock(return_value=False) + + # The run_history has 16 runs + 1 run interruption ==> 16 runs + api.run_history.data = make_dict_run_history_data(run_history_data) + api._metric = accuracy + api.dataset_name = 'iris' + api._scoring_functions = [accuracy, balanced_accuracy] + api.search_space = MagicMock(spec=ConfigurationSpace) + worst_val = api._metric._worst_possible_result + search_results = api.get_search_results() + + _check_status(search_results.status_types) + _check_costs(search_results.opt_scores) + _check_end_times(search_results.end_times) + _check_fit_times(search_results.fit_times) + _check_budgets(search_results.budgets) + _check_metric_dict(search_results.opt_metric_dict, search_results.status_types, worst_val) + _check_additional_infos(status_types=search_results.status_types, + additional_infos=search_results.additional_infos) + + # config_ids can duplicate because of various budget size + config_ids = [1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 1, 10, 11, 12, 10, 13] + assert config_ids == search_results.config_ids + + # assert that contents of search_results are of expected types + assert isinstance(search_results.rank_opt_scores, np.ndarray) + assert search_results.rank_opt_scores.dtype is np.dtype(np.int) + assert isinstance(search_results.configs, list) + + n_success, n_timeout, n_memoryout, n_crashed = 13, 2, 0, 1 + msg = ["autoPyTorch results:", f"\tDataset name: {api.dataset_name}", + f"\tOptimisation Metric: {api._metric.name}", + f"\tBest validation score: {max(search_results.opt_scores)}", + "\tNumber of target algorithm runs: 16", f"\tNumber of successful target algorithm runs: {n_success}", + f"\tNumber of crashed target algorithm runs: {n_crashed}", + f"\tNumber of target algorithms that exceeded the time limit: {n_timeout}", + f"\tNumber of target algorithms that exceeded the memory limit: {n_memoryout}"] + + assert isinstance(api.sprint_statistics(), str) + assert all([m1 == m2 for m1, m2 in zip(api.sprint_statistics().split("\n"), msg)]) + + @pytest.mark.parametrize('run_history', (None, RunHistory())) def test_check_run_history(run_history): manager = ResultsManager() diff --git a/test/test_utils/test_results_visualizer.py b/test/test_utils/test_results_visualizer.py index e31571ef0..4e1bc5951 100644 --- a/test/test_utils/test_results_visualizer.py +++ b/test/test_utils/test_results_visualizer.py @@ -195,6 +195,42 @@ def test_plot_perf_over_time(metric_name): # TODO plt.close() +@pytest.mark.parametrize('metric_name', ('balanced_accuracy', 'accuracy')) +def test_plot_perf_over_time_no_test(metric_name): # TODO + dummy_history = [{'Timestamp': datetime(2022, 1, 1), 'train_accuracy': 1, 'test_accuracy': None}] + BaseTask.__abstractmethods__ = set() + api = BaseTask() + run_history_data = json.load(open(os.path.join(os.path.dirname(__file__), + 'runhistory_no_test.json'), + mode='r'))['data'] + api._results_manager.run_history = MagicMock() + api.run_history.empty = MagicMock(return_value=False) + + # The run_history has 16 runs + 1 run interruption ==> 16 runs + api.run_history.data = make_dict_run_history_data(run_history_data) + api._results_manager.ensemble_performance_history = dummy_history + api._metric = accuracy + api.dataset_name = 'iris' + api._scoring_functions = [accuracy, balanced_accuracy] + api.search_space = MagicMock(spec=ConfigurationSpace) + + api.plot_perf_over_time(metric_name=metric_name) + _, ax = plt.subplots(nrows=1, ncols=1) + api.plot_perf_over_time(metric_name=metric_name, ax=ax) + + # remove ensemble keys if metric name is not for the opt score + ans = set([ + name + for name in [f'single train {metric_name}', + f'single opt {metric_name}', + f'ensemble train {metric_name}'] + if metric_name == api._metric.name or not name.startswith('ensemble') + ]) + legend_set = set([txt._text for txt in ax.get_legend().texts]) + assert ans == legend_set + plt.close() + + @pytest.mark.parametrize('params', ( PlotSettingParams(xscale='none', yscale='none'), PlotSettingParams(xscale='none', yscale='log'), From 0d663f573c7a7a7b629f7395d701c5ae45241e05 Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Mon, 8 Aug 2022 17:26:03 +0200 Subject: [PATCH 2/4] fix flake and test --- autoPyTorch/utils/results_manager.py | 10 ++++++---- autoPyTorch/utils/results_visualizer.py | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/autoPyTorch/utils/results_manager.py b/autoPyTorch/utils/results_manager.py index a374cd2bc..fe7d5acc4 100644 --- a/autoPyTorch/utils/results_manager.py +++ b/autoPyTorch/utils/results_manager.py @@ -1,6 +1,6 @@ import io from datetime import datetime -from typing import Any, Dict, List, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple, Union from ConfigSpace.configuration_space import Configuration @@ -72,7 +72,7 @@ def _extract_metrics_info( run_value: RunValue, scoring_functions: List[autoPyTorchMetric], inference_name: str -) -> Dict[str, float]: +) -> Dict[str, Optional[float]]: """ Extract the metric information given a run_value and a list of metrics of interest. @@ -518,7 +518,8 @@ def _extract_results(self) -> None: data = getattr(self.search_results, f'{inference_name}_metric_dict')[metric_name] if all([d is None for d in data]): if inference_name not in OPTIONAL_INFERENCE_CHOICES: - raise ValueError(f"Expected {metric_name} score for {inference_name} set to not be None, but got {data}") + raise ValueError(f"Expected {metric_name} score for {inference_name} set" + f" to not be None, but got {data}") else: continue self.data[f'single::{inference_name}::{metric_name}'] = np.array(data) @@ -529,7 +530,8 @@ def _extract_results(self) -> None: data = getattr(self.ensemble_results, f'{inference_name}_scores') if all([d is None for d in data]): if inference_name not in OPTIONAL_INFERENCE_CHOICES: - raise ValueError(f"Expected {metric_name} score for {inference_name} set to not be None, but got {data}") + raise ValueError(f"Expected {metric_name} score for {inference_name} set" + f" to not be None, but got {data}") else: continue self.data[f'ensemble::{inference_name}::{metric_name}'] = np.array(data) diff --git a/autoPyTorch/utils/results_visualizer.py b/autoPyTorch/utils/results_visualizer.py index 9853ab7f9..d5d5faa8c 100644 --- a/autoPyTorch/utils/results_visualizer.py +++ b/autoPyTorch/utils/results_visualizer.py @@ -6,7 +6,7 @@ import numpy as np -from autoPyTorch.utils.results_manager import OPTIONAL_INFERENCE_CHOICES, MetricResults +from autoPyTorch.utils.results_manager import MetricResults, OPTIONAL_INFERENCE_CHOICES plt.rcParams["font.family"] = "Times New Roman" From a7934b84aca83ac0041394ef428b1e7bd06e4ba4 Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Tue, 9 Aug 2022 14:01:58 +0200 Subject: [PATCH 3/4] fix bug in tests --- autoPyTorch/utils/results_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autoPyTorch/utils/results_manager.py b/autoPyTorch/utils/results_manager.py index fe7d5acc4..0dc010aeb 100644 --- a/autoPyTorch/utils/results_manager.py +++ b/autoPyTorch/utils/results_manager.py @@ -558,7 +558,7 @@ def get_ensemble_merged_data(self) -> Dict[str, np.ndarray]: cur, timestep_size, sign = 0, self.cum_times.size, self.metric._sign key_train, key_test = f'ensemble::train::{self.metric.name}', f'ensemble::test::{self.metric.name}' - all_test_perfs_null = all([[perf is None for perf in test_scores]]) + all_test_perfs_null = all([perf is None for perf in test_scores]) train_perfs = np.full_like(self.cum_times, self.metric._worst_possible_result) test_perfs = np.full_like(self.cum_times, self.metric._worst_possible_result) From b3d969c673e582bff6735519f3e0cbab2330d2b4 Mon Sep 17 00:00:00 2001 From: Ravin Kohli Date: Fri, 12 Aug 2022 13:35:37 +0200 Subject: [PATCH 4/4] suggestions from review --- autoPyTorch/constants.py | 3 +++ autoPyTorch/utils/results_manager.py | 6 ++---- autoPyTorch/utils/results_visualizer.py | 3 ++- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/autoPyTorch/constants.py b/autoPyTorch/constants.py index bfd56d27f..5934d8c3e 100644 --- a/autoPyTorch/constants.py +++ b/autoPyTorch/constants.py @@ -78,3 +78,6 @@ # To avoid that we get a sequence that is too long to be fed to a network MAX_WINDOW_SIZE_BASE = 500 + +# AutoPyTorch optionally allows network inference or metrics calculation for the following datasets +OPTIONAL_INFERENCE_CHOICES = ('test',) diff --git a/autoPyTorch/utils/results_manager.py b/autoPyTorch/utils/results_manager.py index 0dc010aeb..6547aaef2 100644 --- a/autoPyTorch/utils/results_manager.py +++ b/autoPyTorch/utils/results_manager.py @@ -12,6 +12,7 @@ from smac.tae import StatusType from smac.utils.io.traj_logging import TrajEntry +from autoPyTorch.constants import OPTIONAL_INFERENCE_CHOICES from autoPyTorch.pipeline.components.training.metrics.base import autoPyTorchMetric @@ -28,9 +29,6 @@ ] -OPTIONAL_INFERENCE_CHOICES = ('test',) - - def cost2metric(cost: float, metric: autoPyTorchMetric) -> float: """ Revert cost metric evaluated in SMAC to the original metric. @@ -434,7 +432,7 @@ def _check_null_in_optional_inference_choices( self ) -> None: """ - Checks if the data is missing for each optional inference choice and + Checks if the data is missing or if all the runs failed for each optional inference choice and sets the scores for that inference choice to all None. """ for inference_choice in OPTIONAL_INFERENCE_CHOICES: diff --git a/autoPyTorch/utils/results_visualizer.py b/autoPyTorch/utils/results_visualizer.py index d5d5faa8c..44f931285 100644 --- a/autoPyTorch/utils/results_visualizer.py +++ b/autoPyTorch/utils/results_visualizer.py @@ -6,7 +6,8 @@ import numpy as np -from autoPyTorch.utils.results_manager import MetricResults, OPTIONAL_INFERENCE_CHOICES +from autoPyTorch.constants import OPTIONAL_INFERENCE_CHOICES +from autoPyTorch.utils.results_manager import MetricResults plt.rcParams["font.family"] = "Times New Roman"