From 2d2f6d17da0cd7ba03ba6713f3efad1926516da4 Mon Sep 17 00:00:00 2001 From: nabenabe0928 <47781922+nabenabe0928@users.noreply.github.com> Date: Sun, 21 Nov 2021 09:43:54 +0100 Subject: [PATCH 01/10] [enhance] Increase the coverage (#336) --- test/test_pipeline/test_base_component.py | 89 +++++++++++++++++++ .../test_tabular_classification.py | 11 ++- 2 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 test/test_pipeline/test_base_component.py diff --git a/test/test_pipeline/test_base_component.py b/test/test_pipeline/test_base_component.py new file mode 100644 index 000000000..cf3c25afe --- /dev/null +++ b/test/test_pipeline/test_base_component.py @@ -0,0 +1,89 @@ +from ConfigSpace.configuration_space import ConfigurationSpace +from ConfigSpace.hyperparameters import CategoricalHyperparameter, UniformIntegerHyperparameter + +import pytest + +from autoPyTorch.pipeline.components.base_component import ThirdPartyComponents, autoPyTorchComponent + + +class DummyComponentRequiredFailuire(autoPyTorchComponent): + _required_properties = {'required'} + + def __init__(self, random_state=None): + self.fitted = False + self._cs_updates = {} + + def fit(self, X, y): + self.fitted = True + return self + + def get_properties(dataset_properties=None): + return {"name": 'DummyComponentRequiredFailuire', + "shortname": "Dummy"} + + +class DummyComponentExtraPropFailuire(autoPyTorchComponent): + + def __init__(self, random_state=None): + self.fitted = False + self._cs_updates = {} + + def fit(self, X, y): + self.fitted = True + return self + + def get_properties(dataset_properties=None): + return {"name": 'DummyComponentExtraPropFailuire', + "shortname": 'Dummy', + "must_not_be_there": True} + + +class DummyComponent(autoPyTorchComponent): + def __init__(self, a=0, b='orange', random_state=None): + self.a = a + self.b = b + self.fitted = False + self.random_state = random_state + self._cs_updates = {} + + def get_hyperparameter_search_space(self, dataset_properties=None): + cs = ConfigurationSpace() + a = UniformIntegerHyperparameter('a', lower=10, upper=100, log=False) + b = CategoricalHyperparameter('b', choices=['red', 'green', 'blue']) + cs.add_hyperparameters([a, b]) + return cs + + def fit(self, X, y): + self.fitted = True + return self + + def get_properties(dataset_properties=None): + return {"name": 'DummyComponent', + "shortname": 'Dummy'} + + +def test_third_party_component_failure(): + _addons = ThirdPartyComponents(autoPyTorchComponent) + + with pytest.raises(ValueError, match=r"Property required not specified for .*"): + _addons.add_component(DummyComponentRequiredFailuire) + + with pytest.raises(ValueError, match=r"Property must_not_be_there must not be specified for algorithm .*"): + _addons.add_component(DummyComponentExtraPropFailuire) + + with pytest.raises(TypeError, match=r"add_component works only with a subclass of .*"): + _addons.add_component(1) + + +def test_set_hyperparameters_not_found_failure(): + dummy_component = DummyComponent() + dummy_config_space = dummy_component.get_hyperparameter_search_space() + success_configuration = dummy_config_space.sample_configuration() + dummy_config_space.add_hyperparameter(CategoricalHyperparameter('c', choices=[1, 2])) + failure_configuration = dummy_config_space.sample_configuration() + with pytest.raises(ValueError, match=r"Cannot set hyperparameter c for autoPyTorch.pipeline " + r"DummyComponent because the hyperparameter does not exist."): + dummy_component.set_hyperparameters(failure_configuration) + with pytest.raises(ValueError, match=r"Cannot set init param r for autoPyTorch.pipeline " + r"DummyComponent because the init param does not exist."): + dummy_component.set_hyperparameters(success_configuration, init_params={'r': 1}) diff --git a/test/test_pipeline/test_tabular_classification.py b/test/test_pipeline/test_tabular_classification.py index eb1ffd494..52288b199 100644 --- a/test/test_pipeline/test_tabular_classification.py +++ b/test/test_pipeline/test_tabular_classification.py @@ -13,8 +13,10 @@ import pytest import torch +from torch.optim.lr_scheduler import _LRScheduler from autoPyTorch.pipeline.components.setup.early_preprocessor.utils import get_preprocess_transforms +from autoPyTorch.pipeline.components.setup.lr_scheduler.NoScheduler import NoScheduler from autoPyTorch.pipeline.tabular_classification import TabularClassificationPipeline from autoPyTorch.utils.common import FitRequirement from autoPyTorch.utils.hyperparameter_search_space_update import HyperparameterSearchSpaceUpdates, \ @@ -223,6 +225,7 @@ def test_network_optimizer_lr_handshake(self, fit_dictionary_tabular): # No error when network is passed X = pipeline.named_steps['optimizer'].fit(X, None).transform(X) assert 'optimizer' in X + assert isinstance(pipeline.named_steps['optimizer'].choice.get_optimizer(), torch.optim.Optimizer) # Then fitting a optimizer should fail if no network: assert 'lr_scheduler' in pipeline.named_steps.keys() @@ -234,7 +237,13 @@ def test_network_optimizer_lr_handshake(self, fit_dictionary_tabular): # No error when network is passed X = pipeline.named_steps['lr_scheduler'].fit(X, None).transform(X) - assert 'optimizer' in X + assert 'lr_scheduler' in X + if isinstance(pipeline.named_steps['lr_scheduler'].choice, NoScheduler): + pytest.skip("This scheduler does not support `get_scheduler`") + lr_scheduler = pipeline.named_steps['lr_scheduler'].choice.get_scheduler() + if isinstance(lr_scheduler, torch.optim.lr_scheduler.ReduceLROnPlateau): + pytest.skip("This scheduler is not a child of _LRScheduler") + assert isinstance(lr_scheduler, _LRScheduler) def test_get_fit_requirements(self, fit_dictionary_tabular): dataset_properties = {'numerical_columns': [], 'categorical_columns': [], From 1e06cceda405bb9970ea86bf15f16dbc6c9fbc7f Mon Sep 17 00:00:00 2001 From: nabenabe0928 <47781922+nabenabe0928@users.noreply.github.com> Date: Sun, 21 Nov 2021 19:24:46 +0100 Subject: [PATCH 02/10] [feat] Support statistics print by adding results manager object (#334) * [feat] Support statistics print by adding results manager object * [refactor] Make SearchResults extract run_history at __init__ Since the search results should not be kept in eternally, I made this class to take run_history in __init__ so that we can implicitly call extraction inside. From this change, the call of extraction from outside is not recommended. However, you can still call it from outside and to prevent mixup of the environment, self.clear() will be called. * [fix] Separate those changes into PR#336 * [fix] Fix so that test_loss includes all the metrics * [enhance] Strengthen the test for sprint and SearchResults * [fix] Fix an issue in documentation * [enhance] Increase the coverage * [refactor] Separate the test for results_manager to organize the structure * [test] Add the test for get_incumbent_Result * [test] Remove the previous test_get_incumbent and see the coverage * [fix] [test] Fix reversion of metric and strengthen the test cases * [fix] Fix flake8 issues and increase coverage * [fix] Address Ravin's comments * [enhance] Increase the coverage * [fix] Fix a flake8 issu --- autoPyTorch/api/base_task.py | 98 +- autoPyTorch/api/results_manager.py | 326 +++ autoPyTorch/api/tabular_classification.py | 1 + autoPyTorch/api/tabular_regression.py | 1 + autoPyTorch/evaluation/abstract_evaluator.py | 21 +- .../example_tabular_classification.py | 5 +- .../20_basics/example_tabular_regression.py | 4 +- .../example_custom_configuration_space.py | 8 +- .../example_resampling_strategy.py | 12 +- .../40_advanced/example_run_with_portfolio.py | 4 +- examples/40_advanced/example_visualization.py | 2 +- test/test_api/.tmp_api/runhistory_B.json | 1815 +++++++++++++++++ test/test_api/test_api.py | 71 +- test/test_api/test_results_manager.py | 232 +++ test/test_api/utils.py | 23 +- 15 files changed, 2505 insertions(+), 118 deletions(-) create mode 100644 autoPyTorch/api/results_manager.py create mode 100755 test/test_api/.tmp_api/runhistory_B.json create mode 100644 test/test_api/test_results_manager.py diff --git a/autoPyTorch/api/base_task.py b/autoPyTorch/api/base_task.py index 3973649e7..ab71a761a 100644 --- a/autoPyTorch/api/base_task.py +++ b/autoPyTorch/api/base_task.py @@ -29,6 +29,7 @@ from smac.stats.stats import Stats from smac.tae import StatusType +from autoPyTorch.api.results_manager import ResultsManager, SearchResults from autoPyTorch.automl_common.common.utils.backend import Backend, create from autoPyTorch.constants import ( REGRESSION_TASKS, @@ -192,12 +193,13 @@ def __init__( self.search_space: Optional[ConfigurationSpace] = None self._dataset_requirements: Optional[List[FitRequirement]] = None self._metric: Optional[autoPyTorchMetric] = None + self._scoring_functions: Optional[List[autoPyTorchMetric]] = None self._logger: Optional[PicklableClientLogger] = None - self.run_history: RunHistory = RunHistory() - self.trajectory: Optional[List] = None self.dataset_name: Optional[str] = None self.cv_models_: Dict = {} + self._results_manager = ResultsManager() + # By default try to use the TCP logging port or get a new port self._logger_port = logging.handlers.DEFAULT_TCP_LOGGING_PORT @@ -240,6 +242,18 @@ def build_pipeline(self, dataset_properties: Dict[str, Any]) -> BasePipeline: """ raise NotImplementedError + @property + def run_history(self) -> RunHistory: + return self._results_manager.run_history + + @property + def ensemble_performance_history(self) -> List[Dict[str, Any]]: + return self._results_manager.ensemble_performance_history + + @property + def trajectory(self) -> Optional[List]: + return self._results_manager.trajectory + def set_pipeline_config(self, **pipeline_config_kwargs: Any) -> None: """ Check whether arguments are valid and @@ -883,6 +897,12 @@ def _search( self.pipeline_options['optimize_metric'] = optimize_metric + if all_supported_metrics: + self._scoring_functions = get_metrics(dataset_properties=dataset_properties, + all_supported_metrics=True) + else: + self._scoring_functions = [self._metric] + self.search_space = self.get_search_space(dataset) # Incorporate budget to pipeline config @@ -1037,12 +1057,14 @@ def _search( pynisher_context=self._multiprocessing_context, ) try: - run_history, self.trajectory, budget_type = \ + run_history, self._results_manager.trajectory, budget_type = \ _proc_smac.run_smbo(func=tae_func) self.run_history.update(run_history, DataOrigin.INTERNAL) trajectory_filename = os.path.join( self._backend.get_smac_output_directory_for_run(self.seed), 'trajectory.json') + + assert self.trajectory is not None # mypy check saveable_trajectory = \ [list(entry[:2]) + [entry[2].get_dictionary()] + list(entry[3:]) for entry in self.trajectory] @@ -1059,7 +1081,7 @@ def _search( self._logger.info("Starting Shutdown") if proc_ensemble is not None: - self.ensemble_performance_history = list(proc_ensemble.history) + self._results_manager.ensemble_performance_history = list(proc_ensemble.history) if len(proc_ensemble.futures) > 0: # Also add ensemble runs that did not finish within smac time @@ -1068,7 +1090,7 @@ def _search( result = proc_ensemble.futures.pop().result() if result: ensemble_history, _, _, _ = result - self.ensemble_performance_history.extend(ensemble_history) + self._results_manager.ensemble_performance_history.extend(ensemble_history) self._logger.info("Ensemble script finished, continue shutdown.") # save the ensemble performance history file @@ -1356,28 +1378,12 @@ def get_incumbent_results( The incumbent configuration Dict[str, Union[int, str, float]]: Additional information about the run of the incumbent configuration. - """ - assert self.run_history is not None, "No Run History found, search has not been called." - if self.run_history.empty(): - raise ValueError("Run History is empty. Something went wrong, " - "smac was not able to fit any model?") - - run_history_data = self.run_history.data - if not include_traditional: - # traditional classifiers have trainer_configuration in their additional info - run_history_data = dict( - filter(lambda elem: elem[1].status == StatusType.SUCCESS and elem[1]. - additional_info is not None and elem[1]. - additional_info['configuration_origin'] != 'traditional', - run_history_data.items())) - run_history_data = dict( - filter(lambda elem: 'SUCCESS' in str(elem[1].status), run_history_data.items())) - sorted_runvalue_by_cost = sorted(run_history_data.items(), key=lambda item: item[1].cost) - incumbent_run_key, incumbent_run_value = sorted_runvalue_by_cost[0] - incumbent_config = self.run_history.ids_config[incumbent_run_key.config_id] - incumbent_results = incumbent_run_value.additional_info - return incumbent_config, incumbent_results + + if self._metric is None: + raise RuntimeError("`search_results` is only available after a search has finished.") + + return self._results_manager.get_incumbent_results(metric=self._metric, include_traditional=include_traditional) def get_models_with_weights(self) -> List: if self.models_ is None or len(self.models_) == 0 or \ @@ -1417,3 +1423,43 @@ def _print_debug_info_to_log(self) -> None: self._logger.debug(' multiprocessing_context: %s', str(self._multiprocessing_context)) for key, value in vars(self).items(): self._logger.debug(f"\t{key}->{value}") + + def get_search_results(self) -> SearchResults: + """ + Get the interface to obtain the search results easily. + """ + if self._scoring_functions is None or self._metric is None: + raise RuntimeError("`search_results` is only available after a search has finished.") + + return self._results_manager.get_search_results( + metric=self._metric, + scoring_functions=self._scoring_functions + ) + + def sprint_statistics(self) -> str: + """ + Prints statistics about the SMAC search. + + These statistics include: + + 1. Optimisation Metric + 2. Best Optimisation score achieved by individual pipelines + 3. Total number of target algorithm runs + 4. Total number of successful target algorithm runs + 5. Total number of crashed target algorithm runs + 6. Total number of target algorithm runs that exceeded the time limit + 7. Total number of successful target algorithm runs that exceeded the memory limit + + Returns: + (str): + Formatted string with statistics + """ + if self._scoring_functions is None or self._metric is None: + raise RuntimeError("`search_results` is only available after a search has finished.") + + assert self.dataset_name is not None # my check + return self._results_manager.sprint_statistics( + dataset_name=self.dataset_name, + scoring_functions=self._scoring_functions, + metric=self._metric + ) diff --git a/autoPyTorch/api/results_manager.py b/autoPyTorch/api/results_manager.py new file mode 100644 index 000000000..e52d21613 --- /dev/null +++ b/autoPyTorch/api/results_manager.py @@ -0,0 +1,326 @@ +import io +from typing import Any, Dict, List, Optional, Tuple, Union + +from ConfigSpace.configuration_space import Configuration + +import numpy as np + +import scipy + +from smac.runhistory.runhistory import RunHistory, RunValue +from smac.tae import StatusType +from smac.utils.io.traj_logging import TrajEntry + +from autoPyTorch.pipeline.components.training.metrics.base import autoPyTorchMetric + + +# TODO remove StatusType.RUNNING at some point in the future when the new SMAC 0.13.2 +# is the new minimum required version! +STATUS2MSG = { + StatusType.SUCCESS: 'Success', + StatusType.DONOTADVANCE: 'Success (but did not advance to higher budget)', + StatusType.TIMEOUT: 'Timeout', + StatusType.CRASHED: 'Crash', + StatusType.ABORT: 'Abort', + StatusType.MEMOUT: 'Memory out' +} + + +def cost2metric(cost: float, metric: autoPyTorchMetric) -> float: + """ + Revert cost metric evaluated in SMAC to the original metric. + + The conversion is defined in: + autoPyTorch/pipeline/components/training/metrics/utils.py::calculate_loss + cost = metric._optimum - metric._sign * original_metric_value + ==> original_metric_value = metric._sign * (metric._optimum - cost) + """ + return metric._sign * (metric._optimum - cost) + + +def _extract_metrics_info( + run_value: RunValue, + scoring_functions: List[autoPyTorchMetric] +) -> Dict[str, float]: + """ + Extract the metric information given a run_value + and a list of metrics of interest. + + Args: + run_value (RunValue): + The information for each config evaluation. + scoring_functions (List[autoPyTorchMetric]): + The list of metrics to retrieve the info. + """ + + if run_value.status not in (StatusType.SUCCESS, StatusType.DONOTADVANCE): + # Additional info for metrics is not available in this case. + return {metric.name: np.nan for metric in scoring_functions} + + cost_info = run_value.additional_info['opt_loss'] + avail_metrics = cost_info.keys() + + return { + metric.name: cost2metric(cost=cost_info[metric.name], metric=metric) + if metric.name in avail_metrics else np.nan + for metric in scoring_functions + } + + +class SearchResults: + def __init__( + self, + metric: autoPyTorchMetric, + scoring_functions: List[autoPyTorchMetric], + run_history: RunHistory + ): + self.metric_dict: Dict[str, List[float]] = { + metric.name: [] + for metric in scoring_functions + } + self._opt_scores: List[float] = [] + self._fit_times: List[float] = [] + self.configs: List[Configuration] = [] + self.status_types: List[str] = [] + self.budgets: List[float] = [] + self.config_ids: List[int] = [] + self.is_traditionals: List[bool] = [] + self.additional_infos: List[Optional[Dict[str, Any]]] = [] + self.rank_test_scores: np.ndarray = np.array([]) + self._scoring_functions = scoring_functions + self._metric = metric + + self._extract_results_from_run_history(run_history) + + @property + def opt_scores(self) -> np.ndarray: + return np.asarray(self._opt_scores) + + @property + def fit_times(self) -> np.ndarray: + return np.asarray(self._fit_times) + + def update( + self, + config: Configuration, + status: str, + budget: float, + fit_time: float, + config_id: int, + is_traditional: bool, + additional_info: Dict[str, Any], + score: float, + metric_info: Dict[str, float] + ) -> None: + + self.status_types.append(status) + self.configs.append(config) + self.budgets.append(budget) + self.config_ids.append(config_id) + self.is_traditionals.append(is_traditional) + self.additional_infos.append(additional_info) + self._fit_times.append(fit_time) + self._opt_scores.append(score) + + for metric_name, val in metric_info.items(): + self.metric_dict[metric_name].append(val) + + def clear(self) -> None: + self._opt_scores = [] + self._fit_times = [] + self.configs = [] + self.status_types = [] + self.budgets = [] + self.config_ids = [] + self.additional_infos = [] + self.is_traditionals = [] + self.rank_test_scores = np.array([]) + + def _extract_results_from_run_history(self, run_history: RunHistory) -> None: + """ + Extract the information to match this class format. + + Args: + run_history (RunHistory): + The history of config evals from SMAC. + """ + + self.clear() # Delete cache before the extraction + + for run_key, run_value in run_history.data.items(): + config_id = run_key.config_id + config = run_history.ids_config[config_id] + + status_msg = STATUS2MSG.get(run_value.status, None) + if run_value.status in (StatusType.STOP, StatusType.RUNNING): + continue + elif status_msg is None: + raise ValueError(f'Unexpected run status: {run_value.status}') + + is_traditional = False # If run is not successful, unsure ==> not True ==> False + if run_value.additional_info is not None: + is_traditional = run_value.additional_info['configuration_origin'] == 'traditional' + + self.update( + status=status_msg, + config=config, + budget=run_key.budget, + fit_time=run_value.time, + score=cost2metric(cost=run_value.cost, metric=self._metric), + metric_info=_extract_metrics_info(run_value=run_value, scoring_functions=self._scoring_functions), + is_traditional=is_traditional, + additional_info=run_value.additional_info, + config_id=config_id + ) + + self.rank_test_scores = scipy.stats.rankdata( + -1 * self._metric._sign * self.opt_scores, # rank order + method='min' + ) + + +class ResultsManager: + def __init__(self, *args: Any, **kwargs: Any): + """ + Attributes: + run_history (RunHistory): + A `SMAC Runshistory `_ + object that holds information about the runs of the target algorithm made during search + ensemble_performance_history (List[Dict[str, Any]]): + The list of ensemble performance in the optimization. + The list includes the `timestamp`, `result on train set`, and `result on test set` + trajectory (List[TrajEntry]): + A list of all incumbent configurations during search + """ + self.run_history: RunHistory = RunHistory() + self.ensemble_performance_history: List[Dict[str, Any]] = [] + self.trajectory: List[TrajEntry] = [] + + def _check_run_history(self) -> None: + if self.run_history is None: + raise RuntimeError("No Run History found, search has not been called.") + + if self.run_history.empty(): + raise RuntimeError("Run History is empty. Something went wrong, " + "SMAC was not able to fit any model?") + + def get_incumbent_results( + self, + metric: autoPyTorchMetric, + include_traditional: bool = False + ) -> Tuple[Configuration, Dict[str, Union[int, str, float]]]: + """ + Get Incumbent config and the corresponding results + + Args: + metric (autoPyTorchMetric): + A metric that is evaluated when searching with fit AutoPytorch. + include_traditional (bool): + Whether to include results from tradtional pipelines + + Returns: + Configuration (CS.ConfigurationSpace): + The incumbent configuration + Dict[str, Union[int, str, float]]: + Additional information about the run of the incumbent configuration. + """ + self._check_run_history() + + results = SearchResults(metric=metric, scoring_functions=[], run_history=self.run_history) + + if not include_traditional: + non_traditional = ~np.array(results.is_traditionals) + scores = results.opt_scores[non_traditional] + indices = np.arange(len(results.configs))[non_traditional] + else: + scores = results.opt_scores + indices = np.arange(len(results.configs)) + + incumbent_idx = indices[np.nanargmax(metric._sign * scores)] + incumbent_config = results.configs[incumbent_idx] + incumbent_results = results.additional_infos[incumbent_idx] + + assert incumbent_results is not None # mypy check + return incumbent_config, incumbent_results + + def get_search_results( + self, + scoring_functions: List[autoPyTorchMetric], + metric: autoPyTorchMetric + ) -> SearchResults: + """ + This attribute is populated with data from `self.run_history` + and contains information about the configurations, and their + corresponding metric results, status of run, parameters and + the budget + + Args: + scoring_functions (List[autoPyTorchMetric]): + Metrics to show in the results. + metric (autoPyTorchMetric): + A metric that is evaluated when searching with fit AutoPytorch. + + Returns: + SearchResults: + An instance that contains the results from search + """ + self._check_run_history() + return SearchResults(metric=metric, scoring_functions=scoring_functions, run_history=self.run_history) + + def sprint_statistics( + self, + dataset_name: str, + scoring_functions: List[autoPyTorchMetric], + metric: autoPyTorchMetric + ) -> str: + """ + Prints statistics about the SMAC search. + + These statistics include: + + 1. Optimisation Metric + 2. Best Optimisation score achieved by individual pipelines + 3. Total number of target algorithm runs + 4. Total number of successful target algorithm runs + 5. Total number of crashed target algorithm runs + 6. Total number of target algorithm runs that exceeded the time limit + 7. Total number of successful target algorithm runs that exceeded the memory limit + + Args: + dataset_name (str): + The dataset name that was used in the run. + scoring_functions (List[autoPyTorchMetric]): + Metrics to show in the results. + metric (autoPyTorchMetric): + A metric that is evaluated when searching with fit AutoPytorch. + + Returns: + (str): + Formatted string with statistics + """ + search_results = self.get_search_results(scoring_functions, metric) + success_msgs = (STATUS2MSG[StatusType.SUCCESS], STATUS2MSG[StatusType.DONOTADVANCE]) + sio = io.StringIO() + sio.write("autoPyTorch results:\n") + sio.write(f"\tDataset name: {dataset_name}\n") + sio.write(f"\tOptimisation Metric: {metric}\n") + + num_runs = len(search_results.status_types) + num_success = sum([s in success_msgs for s in search_results.status_types]) + num_crash = sum([s == STATUS2MSG[StatusType.CRASHED] for s in search_results.status_types]) + num_timeout = sum([s == STATUS2MSG[StatusType.TIMEOUT] for s in search_results.status_types]) + num_memout = sum([s == STATUS2MSG[StatusType.MEMOUT] for s in search_results.status_types]) + + if num_success > 0: + best_score = metric._sign * np.nanmax(metric._sign * search_results.opt_scores) + sio.write(f"\tBest validation score: {best_score}\n") + + sio.write(f"\tNumber of target algorithm runs: {num_runs}\n") + sio.write(f"\tNumber of successful target algorithm runs: {num_success}\n") + sio.write(f"\tNumber of crashed target algorithm runs: {num_crash}\n") + sio.write(f"\tNumber of target algorithms that exceeded the time " + f"limit: {num_timeout}\n") + sio.write(f"\tNumber of target algorithms that exceeded the memory " + f"limit: {num_memout}\n") + + return sio.getvalue() diff --git a/autoPyTorch/api/tabular_classification.py b/autoPyTorch/api/tabular_classification.py index 59d67ae5c..1e73945c3 100644 --- a/autoPyTorch/api/tabular_classification.py +++ b/autoPyTorch/api/tabular_classification.py @@ -275,6 +275,7 @@ def search( X=X_train, Y=y_train, X_test=X_test, Y_test=y_test, validator=self.InputValidator, + dataset_name=dataset_name, resampling_strategy=self.resampling_strategy, resampling_strategy_args=self.resampling_strategy_args, ) diff --git a/autoPyTorch/api/tabular_regression.py b/autoPyTorch/api/tabular_regression.py index d94589638..6f0f22d24 100644 --- a/autoPyTorch/api/tabular_regression.py +++ b/autoPyTorch/api/tabular_regression.py @@ -276,6 +276,7 @@ def search( X=X_train, Y=y_train, X_test=X_test, Y_test=y_test, validator=self.InputValidator, + dataset_name=dataset_name, resampling_strategy=self.resampling_strategy, resampling_strategy_args=self.resampling_strategy_args, ) diff --git a/autoPyTorch/evaluation/abstract_evaluator.py b/autoPyTorch/evaluation/abstract_evaluator.py index b926a50a7..027c7211a 100644 --- a/autoPyTorch/evaluation/abstract_evaluator.py +++ b/autoPyTorch/evaluation/abstract_evaluator.py @@ -735,7 +735,7 @@ def calculate_auxiliary_losses( self, Y_valid_pred: np.ndarray, Y_test_pred: np.ndarray, - ) -> Tuple[Optional[float], Optional[float]]: + ) -> Tuple[Optional[Dict[str, float]], Optional[Dict[str, float]]]: """ A helper function to calculate the performance estimate of the current pipeline in the user provided validation/test set. @@ -747,29 +747,26 @@ def calculate_auxiliary_losses( Y_test_pred (np.ndarray): predictions on a test set provided by the user, matching self.y_test + Returns: - validation_loss (Optional[float]): - The validation loss under the optimization metric - stored in self.metric - test_loss (Optional[float]]): - The test loss under the optimization metric - stored in self.metric + validation_loss_dict (Optional[Dict[str, float]]): + Various validation losses available. + test_loss_dict (Optional[Dict[str, float]]): + Various test losses available. """ - validation_loss: Optional[float] = None + validation_loss_dict: Optional[Dict[str, float]] = None if Y_valid_pred is not None: if self.y_valid is not None: validation_loss_dict = self._loss(self.y_valid, Y_valid_pred) - validation_loss = validation_loss_dict[self.metric.name] - test_loss: Optional[float] = None + test_loss_dict: Optional[Dict[str, float]] = None if Y_test_pred is not None: if self.y_test is not None: test_loss_dict = self._loss(self.y_test, Y_test_pred) - test_loss = test_loss_dict[self.metric.name] - return validation_loss, test_loss + return validation_loss_dict, test_loss_dict def file_output( self, diff --git a/examples/20_basics/example_tabular_classification.py b/examples/20_basics/example_tabular_classification.py index 7b1aa9995..636281eff 100644 --- a/examples/20_basics/example_tabular_classification.py +++ b/examples/20_basics/example_tabular_classification.py @@ -55,6 +55,7 @@ y_train=y_train, X_test=X_test.copy(), y_test=y_test.copy(), + dataset_name='Australian', optimize_metric='accuracy', total_walltime_limit=300, func_eval_time_limit_secs=50 @@ -63,9 +64,11 @@ ############################################################################ # Print the final ensemble performance # ==================================== -print(api.run_history, api.trajectory) y_pred = api.predict(X_test) score = api.score(y_pred, y_test) print(score) # Print the final ensemble built by AutoPyTorch print(api.show_models()) + +# Print statistics from search +print(api.sprint_statistics()) diff --git a/examples/20_basics/example_tabular_regression.py b/examples/20_basics/example_tabular_regression.py index 9b4e876e9..127f26829 100644 --- a/examples/20_basics/example_tabular_regression.py +++ b/examples/20_basics/example_tabular_regression.py @@ -55,7 +55,6 @@ ############################################################################ # Print the final ensemble performance # ==================================== -print(api.run_history, api.trajectory) y_pred = api.predict(X_test) # Rescale the Neural Network predictions into the original target range @@ -64,3 +63,6 @@ print(score) # Print the final ensemble built by AutoPyTorch print(api.show_models()) + +# Print statistics from search +print(api.sprint_statistics()) diff --git a/examples/40_advanced/example_custom_configuration_space.py b/examples/40_advanced/example_custom_configuration_space.py index f552045c1..c64a4fca1 100644 --- a/examples/40_advanced/example_custom_configuration_space.py +++ b/examples/40_advanced/example_custom_configuration_space.py @@ -94,12 +94,14 @@ def get_search_space_updates(): ############################################################################ # Print the final ensemble performance # ==================================== - print(api.run_history, api.trajectory) y_pred = api.predict(X_test) score = api.score(y_pred, y_test) print(score) print(api.show_models()) + # Print statistics from search + print(api.sprint_statistics()) + ############################################################################ # Build and fit a classifier with exclude components # ================================================== @@ -125,8 +127,10 @@ def get_search_space_updates(): ############################################################################ # Print the final ensemble performance # ==================================== - print(api.run_history, api.trajectory) y_pred = api.predict(X_test) score = api.score(y_pred, y_test) print(score) print(api.show_models()) + + # Print statistics from search + print(api.sprint_statistics()) diff --git a/examples/40_advanced/example_resampling_strategy.py b/examples/40_advanced/example_resampling_strategy.py index 9fb77b76d..6735fffee 100644 --- a/examples/40_advanced/example_resampling_strategy.py +++ b/examples/40_advanced/example_resampling_strategy.py @@ -65,13 +65,15 @@ ############################################################################ # Print the final ensemble performance # ==================================== -print(api.run_history, api.trajectory) y_pred = api.predict(X_test) score = api.score(y_pred, y_test) print(score) # Print the final ensemble built by AutoPyTorch print(api.show_models()) +# Print statistics from search +print(api.sprint_statistics()) + ############################################################################ ############################################################################ @@ -98,13 +100,15 @@ ############################################################################ # Print the final ensemble performance # ==================================== -print(api.run_history, api.trajectory) y_pred = api.predict(X_test) score = api.score(y_pred, y_test) print(score) # Print the final ensemble built by AutoPyTorch print(api.show_models()) +# Print statistics from search +print(api.sprint_statistics()) + ############################################################################ ############################################################################ @@ -134,9 +138,11 @@ ############################################################################ # Print the final ensemble performance # ==================================== -print(api.run_history, api.trajectory) y_pred = api.predict(X_test) score = api.score(y_pred, y_test) print(score) # Print the final ensemble built by AutoPyTorch print(api.show_models()) + +# Print statistics from search +print(api.sprint_statistics()) diff --git a/examples/40_advanced/example_run_with_portfolio.py b/examples/40_advanced/example_run_with_portfolio.py index 4109a2378..01d8bef15 100644 --- a/examples/40_advanced/example_run_with_portfolio.py +++ b/examples/40_advanced/example_run_with_portfolio.py @@ -63,9 +63,11 @@ ############################################################################ # Print the final ensemble performance # ==================================== - print(api.run_history, api.trajectory) y_pred = api.predict(X_test) score = api.score(y_pred, y_test) print(score) # Print the final ensemble built by AutoPyTorch print(api.show_models()) + + # Print statistics from search + print(api.sprint_statistics()) diff --git a/examples/40_advanced/example_visualization.py b/examples/40_advanced/example_visualization.py index 107d07a47..37c1c6dc3 100644 --- a/examples/40_advanced/example_visualization.py +++ b/examples/40_advanced/example_visualization.py @@ -113,7 +113,7 @@ ), 'single_best_optimization_accuracy': accuracy._optimum - run_value.cost, 'single_best_test_accuracy': np.nan if run_value.additional_info is None else - accuracy._optimum - run_value.additional_info['test_loss'], + accuracy._optimum - run_value.additional_info['test_loss']['accuracy'], }) individual_performance_frame = pd.DataFrame(individual_performances) diff --git a/test/test_api/.tmp_api/runhistory_B.json b/test/test_api/.tmp_api/runhistory_B.json new file mode 100755 index 000000000..37e499664 --- /dev/null +++ b/test/test_api/.tmp_api/runhistory_B.json @@ -0,0 +1,1815 @@ +{ + "data": [ + [ + [ + 1, + "{\"task_id\": \"Australian\"}", + 0, + 5.555555555555555 + ], + [ + 0.15204678362573099, + 3.154788017272949, + { + "__enum__": "StatusType.SUCCESS" + }, + 1637342638.6119366, + 1637342642.7887495, + { + "opt_loss": { + "accuracy": 0.15204678362573099, + "balanced_accuracy": 0.15263157894736845, + "roc_auc": 0.08981994459833786, + "average_precision": 0.1040861796433199, + "log_loss": 0.5765479137672738, + "precision": 0.17948717948717952, + "precision_macro": 0.15425971877584788, + "precision_micro": 0.15204678362573099, + "precision_weighted": 0.15145666758569976, + "recall": 0.1578947368421053, + "recall_macro": 0.15263157894736845, + "recall_micro": 0.15204678362573099, + "recall_weighted": 0.15204678362573099, + "f1": 0.16883116883116878, + "f1_macro": 0.15356452058579717, + "f1_micro": 0.15204678362573099, + "f1_weighted": 0.15186822633631147 + }, + "duration": 3.11077618598938, + "num_run": 8, + "train_loss": { + "accuracy": 0.09537572254335258, + "balanced_accuracy": 0.10239948774980623, + "roc_auc": 0.03963198867657458, + "average_precision": 0.044469547423341305, + "log_loss": 0.28008669264774966, + "precision": 0.03731343283582089, + "precision_macro": 0.08469445226696704, + "precision_micro": 0.09537572254335258, + "precision_weighted": 0.0890765118675354, + "recall": 0.17834394904458595, + "recall_macro": 0.10239948774980623, + "recall_micro": 0.09537572254335258, + "recall_weighted": 0.09537572254335258, + "f1": 0.11340206185567014, + "f1_macro": 0.09784816309741107, + "f1_micro": 0.09537572254335258, + "f1_weighted": 0.0964096522295953 + }, + "test_loss": { + "accuracy": 0.1445086705202312, + "balanced_accuracy": 0.15356265356265353, + "roc_auc": 0.07821457821457822, + "average_precision": 0.08337624010373501, + "log_loss": 0.4291606295042948, + "precision": 0.13432835820895528, + "precision_macro": 0.14263587721768523, + "precision_micro": 0.1445086705202312, + "precision_weighted": 0.14383638574495827, + "recall": 0.21621621621621623, + "recall_macro": 0.15356265356265353, + "recall_micro": 0.1445086705202312, + "recall_weighted": 0.1445086705202312, + "f1": 0.17730496453900713, + "f1_macro": 0.14962809202560112, + "f1_micro": 0.1445086705202313, + "f1_weighted": 0.14562854397453084 + }, + "configuration_origin": "Default" + } + ] + ], + [ + [ + 2, + "{\"task_id\": \"Australian\"}", + 0, + 5.555555555555555 + ], + [ + 0.4444444444444444, + 3.2763524055480957, + { + "__enum__": "StatusType.SUCCESS" + }, + 1637342642.963385, + 1637342647.2651122, + { + "opt_loss": { + "accuracy": 0.4444444444444444, + "balanced_accuracy": 0.5, + "roc_auc": 0.25526315789473697, + "average_precision": 0.35005634879129066, + "log_loss": 1.0913122792494052, + "precision": 1.0, + "precision_macro": 0.7222222222222222, + "precision_micro": 0.4444444444444444, + "precision_weighted": 0.691358024691358, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.4444444444444444, + "recall_weighted": 0.4444444444444444, + "f1": 1.0, + "f1_macro": 0.6428571428571428, + "f1_micro": 0.4444444444444444, + "f1_weighted": 0.6031746031746031 + }, + "duration": 3.2138161659240723, + "num_run": 9, + "train_loss": { + "accuracy": 0.45375722543352603, + "balanced_accuracy": 0.5, + "roc_auc": 0.2745256630606949, + "average_precision": 0.4037230365622788, + "log_loss": 1.1229484684905306, + "precision": 1.0, + "precision_macro": 0.726878612716763, + "precision_micro": 0.45375722543352603, + "precision_weighted": 0.7016188312339203, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.45375722543352603, + "recall_weighted": 0.45375722543352603, + "f1": 1.0, + "f1_macro": 0.6467289719626168, + "f1_micro": 0.45375722543352603, + "f1_weighted": 0.6140565069418183 + }, + "test_loss": { + "accuracy": 0.4277456647398844, + "balanced_accuracy": 0.5, + "roc_auc": 0.28078078078078084, + "average_precision": 0.4240469510154605, + "log_loss": 1.0714287830118328, + "precision": 1.0, + "precision_macro": 0.7138728323699421, + "precision_micro": 0.4277456647398844, + "precision_weighted": 0.6725249757760032, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.4277456647398844, + "recall_weighted": 0.4277456647398844, + "f1": 1.0, + "f1_macro": 0.6360294117647058, + "f1_micro": 0.4277456647398844, + "f1_weighted": 0.5834325059503571 + }, + "configuration_origin": "Random Search" + } + ] + ], + [ + [ + 3, + "{\"task_id\": \"Australian\"}", + 0, + 5.555555555555555 + ], + [ + 0.5555555555555556, + 22.723600149154663, + { + "__enum__": "StatusType.SUCCESS" + }, + 1637342651.4707444, + 1637342675.2555833, + { + "opt_loss": { + "accuracy": 0.5555555555555556, + "balanced_accuracy": 0.5, + "roc_auc": 0.4924515235457063, + "average_precision": 0.5493808049535605, + "log_loss": 0.7291908971747459, + "precision": 0.5555555555555556, + "precision_macro": 0.7777777777777778, + "precision_micro": 0.5555555555555556, + "precision_weighted": 0.8024691358024691, + "recall": 0.0, + "recall_macro": 0.5, + "recall_micro": 0.5555555555555556, + "recall_weighted": 0.5555555555555556, + "f1": 0.3846153846153847, + "f1_macro": 0.6923076923076923, + "f1_micro": 0.5555555555555556, + "f1_weighted": 0.7264957264957266 + }, + "duration": 22.021637201309204, + "num_run": 10, + "train_loss": { + "accuracy": 0.546242774566474, + "balanced_accuracy": 0.5, + "roc_auc": 0.514423887035352, + "average_precision": 0.5521926852639938, + "log_loss": 0.7258427792546377, + "precision": 0.546242774566474, + "precision_macro": 0.773121387283237, + "precision_micro": 0.546242774566474, + "precision_weighted": 0.7941043803668683, + "recall": 0.0, + "recall_macro": 0.5, + "recall_micro": 0.546242774566474, + "recall_weighted": 0.546242774566474, + "f1": 0.3757455268389662, + "f1_macro": 0.6878727634194831, + "f1_micro": 0.546242774566474, + "f1_weighted": 0.7167400222939817 + }, + "test_loss": { + "accuracy": 0.5722543352601156, + "balanced_accuracy": 0.5, + "roc_auc": 0.4951542451542452, + "average_precision": 0.5698669806352692, + "log_loss": 0.7351944336312355, + "precision": 0.5722543352601156, + "precision_macro": 0.7861271676300579, + "precision_micro": 0.5722543352601156, + "precision_weighted": 0.8170336462962344, + "recall": 0.0, + "recall_macro": 0.5, + "recall_micro": 0.5722543352601156, + "recall_weighted": 0.5722543352601156, + "f1": 0.4008097165991902, + "f1_macro": 0.7004048582995951, + "f1_micro": 0.5722543352601157, + "f1_weighted": 0.7436989539210408 + }, + "configuration_origin": "Random Search (sorted)" + } + ] + ], + [ + [ + 4, + "{\"task_id\": \"Australian\"}", + 0, + 5.555555555555555 + ], + [ + 0.29824561403508776, + 4.990685224533081, + { + "__enum__": "StatusType.SUCCESS" + }, + 1637342675.317421, + 1637342681.334954, + { + "opt_loss": { + "accuracy": 0.29824561403508776, + "balanced_accuracy": 0.30263157894736836, + "roc_auc": 0.26869806094182835, + "average_precision": 0.3191125709864897, + "log_loss": 0.6374789248084465, + "precision": 0.33333333333333337, + "precision_macro": 0.30208333333333337, + "precision_micro": 0.29824561403508776, + "precision_weighted": 0.29861111111111116, + "recall": 0.3421052631578947, + "recall_macro": 0.30263157894736836, + "recall_micro": 0.29824561403508776, + "recall_weighted": 0.29824561403508776, + "f1": 0.3377483443708609, + "f1_macro": 0.30238202558857186, + "f1_micro": 0.29824561403508776, + "f1_weighted": 0.29845243461276183 + }, + "duration": 4.924501419067383, + "num_run": 11, + "train_loss": { + "accuracy": 0.3728323699421965, + "balanced_accuracy": 0.3800930138509757, + "roc_auc": 0.3314460957773059, + "average_precision": 0.3638537658311296, + "log_loss": 0.6533903728503023, + "precision": 0.4014084507042254, + "precision_macro": 0.3771748135874068, + "precision_micro": 0.3728323699421965, + "precision_weighted": 0.3749335523511692, + "recall": 0.4585987261146497, + "recall_macro": 0.3800930138509757, + "recall_micro": 0.3728323699421965, + "recall_weighted": 0.3728323699421965, + "f1": 0.43143812709030094, + "f1_macro": 0.3798412009497306, + "f1_micro": 0.3728323699421965, + "f1_weighted": 0.3750692309020478 + }, + "test_loss": { + "accuracy": 0.34104046242774566, + "balanced_accuracy": 0.35087360087360087, + "roc_auc": 0.29060879060879063, + "average_precision": 0.3873731449553385, + "log_loss": 0.6454486294805659, + "precision": 0.3943661971830986, + "precision_macro": 0.34914388290527487, + "precision_micro": 0.34104046242774566, + "precision_weighted": 0.3426088663911384, + "recall": 0.41891891891891897, + "recall_macro": 0.35087360087360087, + "recall_micro": 0.34104046242774566, + "recall_weighted": 0.34104046242774566, + "f1": 0.40689655172413797, + "f1_macro": 0.35021444501629784, + "f1_micro": 0.34104046242774566, + "f1_weighted": 0.34202338913366204 + }, + "configuration_origin": "Random Search" + } + ] + ], + [ + [ + 5, + "{\"task_id\": \"Australian\"}", + 0, + 5.555555555555555 + ], + [ + 0.4444444444444444, + 10.684926509857178, + { + "__enum__": "StatusType.SUCCESS" + }, + 1637342681.548915, + 1637342693.2717755, + { + "opt_loss": { + "accuracy": 0.4444444444444444, + "balanced_accuracy": 0.5, + "roc_auc": 0.6092797783933518, + "average_precision": 0.6129755132627962, + "log_loss": 0.6905045174715811, + "precision": 1.0, + "precision_macro": 0.7222222222222222, + "precision_micro": 0.4444444444444444, + "precision_weighted": 0.691358024691358, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.4444444444444444, + "recall_weighted": 0.4444444444444444, + "f1": 1.0, + "f1_macro": 0.6428571428571428, + "f1_micro": 0.4444444444444444, + "f1_weighted": 0.6031746031746031 + }, + "duration": 10.401196956634521, + "num_run": 12, + "train_loss": { + "accuracy": 0.45375722543352603, + "balanced_accuracy": 0.5, + "roc_auc": 0.6309102551140767, + "average_precision": 0.6325768698403712, + "log_loss": 0.691941062839045, + "precision": 1.0, + "precision_macro": 0.726878612716763, + "precision_micro": 0.45375722543352603, + "precision_weighted": 0.7016188312339203, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.45375722543352603, + "recall_weighted": 0.45375722543352603, + "f1": 1.0, + "f1_macro": 0.6467289719626168, + "f1_micro": 0.45375722543352603, + "f1_weighted": 0.6140565069418183 + }, + "test_loss": { + "accuracy": 0.4450867052023122, + "balanced_accuracy": 0.5151515151515151, + "roc_auc": 0.7676767676767677, + "average_precision": 0.7025371518831428, + "log_loss": 0.6913509909817249, + "precision": 1.0, + "precision_macro": 0.7176470588235294, + "precision_micro": 0.4450867052023122, + "precision_weighted": 0.6768446106766406, + "recall": 1.0, + "recall_macro": 0.5151515151515151, + "recall_micro": 0.4450867052023122, + "recall_weighted": 0.4450867052023122, + "f1": 1.0, + "f1_macro": 0.6431226765799256, + "f1_micro": 0.4450867052023122, + "f1_weighted": 0.5915508090336722 + }, + "configuration_origin": "Random Search" + } + ] + ], + [ + [ + 6, + "{\"task_id\": \"Australian\"}", + 0, + 5.555555555555555 + ], + [ + 0.4444444444444444, + 9.947429180145264, + { + "__enum__": "StatusType.SUCCESS" + }, + 1637342693.356699, + 1637342704.341065, + { + "opt_loss": { + "accuracy": 0.4444444444444444, + "balanced_accuracy": 0.5, + "roc_auc": 0.24930747922437668, + "average_precision": 0.31612650360994055, + "log_loss": 0.6525155201292875, + "precision": 1.0, + "precision_macro": 0.7222222222222222, + "precision_micro": 0.4444444444444444, + "precision_weighted": 0.691358024691358, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.4444444444444444, + "recall_weighted": 0.4444444444444444, + "f1": 1.0, + "f1_macro": 0.6428571428571428, + "f1_micro": 0.4444444444444444, + "f1_weighted": 0.6031746031746031 + }, + "duration": 9.76927137374878, + "num_run": 13, + "train_loss": { + "accuracy": 0.45375722543352603, + "balanced_accuracy": 0.5, + "roc_auc": 0.22427796313146642, + "average_precision": 0.2451792573360162, + "log_loss": 0.64721482587343, + "precision": 1.0, + "precision_macro": 0.726878612716763, + "precision_micro": 0.45375722543352603, + "precision_weighted": 0.7016188312339203, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.45375722543352603, + "recall_weighted": 0.45375722543352603, + "f1": 1.0, + "f1_macro": 0.6467289719626168, + "f1_micro": 0.45375722543352603, + "f1_weighted": 0.6140565069418183 + }, + "test_loss": { + "accuracy": 0.4277456647398844, + "balanced_accuracy": 0.5, + "roc_auc": 0.21730821730821726, + "average_precision": 0.24726417906930265, + "log_loss": 0.6388118587477358, + "precision": 1.0, + "precision_macro": 0.7138728323699421, + "precision_micro": 0.4277456647398844, + "precision_weighted": 0.6725249757760032, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.4277456647398844, + "recall_weighted": 0.4277456647398844, + "f1": 1.0, + "f1_macro": 0.6360294117647058, + "f1_micro": 0.4277456647398844, + "f1_weighted": 0.5834325059503571 + }, + "configuration_origin": "Random Search" + } + ] + ], + [ + [ + 7, + "{\"task_id\": \"Australian\"}", + 0, + 5.555555555555555 + ], + [ + 1.0, + 11.687273979187012, + { + "__enum__": "StatusType.CRASHED" + }, + 1637342713.4931705, + 1637342726.1866672, + { + "error": "Result queue is empty", + "exit_status": "", + "subprocess_stdout": "", + "subprocess_stderr": "Process pynisher function call:\nTraceback (most recent call last):\n File \"/home/shuhei/research/Auto-PyTorch/autoPyTorch/evaluation/tae.py\", line 39, in fit_predict_try_except_decorator\n ta(queue=queue, **kwargs)\n File \"/home/shuhei/research/Auto-PyTorch/autoPyTorch/evaluation/train_evaluator.py\", line 485, in eval_function\n evaluator.fit_predict_and_loss()\n File \"/home/shuhei/research/Auto-PyTorch/autoPyTorch/evaluation/train_evaluator.py\", line 163, in fit_predict_and_loss\n y_train_pred, y_opt_pred, y_valid_pred, y_test_pred = self._fit_and_predict(pipeline, split_id,\n File \"/home/shuhei/research/Auto-PyTorch/autoPyTorch/evaluation/train_evaluator.py\", line 337, in _fit_and_predict\n fit_and_suppress_warnings(self.logger, pipeline, X, y)\n File \"/home/shuhei/research/Auto-PyTorch/autoPyTorch/evaluation/abstract_evaluator.py\", line 321, in fit_and_suppress_warnings\n pipeline.fit(X, y)\n File \"/home/shuhei/research/Auto-PyTorch/autoPyTorch/pipeline/base_pipeline.py\", line 153, in fit\n self.fit_estimator(X, y, **fit_params)\n File \"/home/shuhei/research/Auto-PyTorch/autoPyTorch/pipeline/base_pipeline.py\", line 172, in fit_estimator\n self._final_estimator.fit(X, y, **fit_params)\n File \"/home/shuhei/research/Auto-PyTorch/autoPyTorch/pipeline/components/training/trainer/__init__.py\", line 211, in fit\n self._fit(\n File \"/home/shuhei/research/Auto-PyTorch/autoPyTorch/pipeline/components/training/trainer/__init__.py\", line 290, in _fit\n train_loss, train_metrics = self.choice.train_epoch(\n File \"/home/shuhei/research/Auto-PyTorch/autoPyTorch/pipeline/components/training/trainer/base_trainer.py\", line 303, in train_epoch\n loss, outputs = self.train_step(data, targets)\n File \"/home/shuhei/research/Auto-PyTorch/autoPyTorch/pipeline/components/training/trainer/base_trainer.py\", line 357, in train_step\n outputs = self.model(data)\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 727, in _call_impl\n result = self.forward(*input, **kwargs)\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/site-packages/torch/nn/modules/container.py\", line 117, in forward\n input = module(input)\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 727, in _call_impl\n result = self.forward(*input, **kwargs)\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/site-packages/torch/nn/modules/container.py\", line 117, in forward\n input = module(input)\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 727, in _call_impl\n result = self.forward(*input, **kwargs)\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/site-packages/torch/nn/modules/container.py\", line 117, in forward\n input = module(input)\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 727, in _call_impl\n result = self.forward(*input, **kwargs)\n File \"/home/shuhei/research/Auto-PyTorch/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py\", line 274, in forward\n x2 = self.shake_shake_layers(x)\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 727, in _call_impl\n result = self.forward(*input, **kwargs)\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/site-packages/torch/nn/modules/container.py\", line 117, in forward\n input = module(input)\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 727, in _call_impl\n result = self.forward(*input, **kwargs)\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/site-packages/torch/nn/modules/linear.py\", line 93, in forward\n return F.linear(input, self.weight, self.bias)\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/site-packages/torch/nn/functional.py\", line 1690, in linear\n ret = torch.addmm(bias, input, weight.t())\nRuntimeError: [enforce fail at CPUAllocator.cpp:65] . DefaultCPUAllocator: can't allocate memory: you tried to allocate 713632 bytes. Error code 12 (Cannot allocate memory)\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/multiprocessing/process.py\", line 315, in _bootstrap\n self.run()\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/multiprocessing/process.py\", line 108, in run\n self._target(*self._args, **self._kwargs)\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/site-packages/pynisher/limit_function_call.py\", line 138, in subprocess_func\n return_value = ((func(*args, **kwargs), 0))\n File \"/home/shuhei/research/Auto-PyTorch/autoPyTorch/evaluation/tae.py\", line 52, in fit_predict_try_except_decorator\n queue.put({'loss': cost_for_crash,\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/multiprocessing/queues.py\", line 88, in put\n self._start_thread()\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/multiprocessing/queues.py\", line 173, in _start_thread\n self._thread.start()\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/threading.py\", line 852, in start\n _start_new_thread(self._bootstrap, ())\nRuntimeError: can't start new thread\n", + "exitcode": 1, + "configuration_origin": "Random Search (sorted)" + } + ] + ], + [ + [ + 8, + "{\"task_id\": \"Australian\"}", + 0, + 5.555555555555555 + ], + [ + 0.5555555555555556, + 8.478890419006348, + { + "__enum__": "StatusType.SUCCESS" + }, + 1637342733.815657, + 1637342743.3274522, + { + "opt_loss": { + "accuracy": 0.5555555555555556, + "balanced_accuracy": 0.5, + "roc_auc": 0.44605263157894737, + "average_precision": 0.526907034743722, + "log_loss": 0.722785997111895, + "precision": 0.5555555555555556, + "precision_macro": 0.7777777777777778, + "precision_micro": 0.5555555555555556, + "precision_weighted": 0.8024691358024691, + "recall": 0.0, + "recall_macro": 0.5, + "recall_micro": 0.5555555555555556, + "recall_weighted": 0.5555555555555556, + "f1": 0.3846153846153847, + "f1_macro": 0.6923076923076923, + "f1_micro": 0.5555555555555556, + "f1_weighted": 0.7264957264957266 + }, + "duration": 8.288825988769531, + "num_run": 15, + "train_loss": { + "accuracy": 0.546242774566474, + "balanced_accuracy": 0.5, + "roc_auc": 0.4537121288713646, + "average_precision": 0.5218043063878082, + "log_loss": 0.7198673617633092, + "precision": 0.546242774566474, + "precision_macro": 0.773121387283237, + "precision_micro": 0.546242774566474, + "precision_weighted": 0.7941043803668683, + "recall": 0.0, + "recall_macro": 0.5, + "recall_micro": 0.546242774566474, + "recall_weighted": 0.546242774566474, + "f1": 0.3757455268389662, + "f1_macro": 0.6878727634194831, + "f1_micro": 0.546242774566474, + "f1_weighted": 0.7167400222939817 + }, + "test_loss": { + "accuracy": 0.5722543352601156, + "balanced_accuracy": 0.5, + "roc_auc": 0.49529074529074535, + "average_precision": 0.5699331879028938, + "log_loss": 0.7280194180549224, + "precision": 0.5722543352601156, + "precision_macro": 0.7861271676300579, + "precision_micro": 0.5722543352601156, + "precision_weighted": 0.8170336462962344, + "recall": 0.0, + "recall_macro": 0.5, + "recall_micro": 0.5722543352601156, + "recall_weighted": 0.5722543352601156, + "f1": 0.4008097165991902, + "f1_macro": 0.7004048582995951, + "f1_micro": 0.5722543352601157, + "f1_weighted": 0.7436989539210408 + }, + "configuration_origin": "Random Search (sorted)" + } + ] + ], + [ + [ + 9, + "{\"task_id\": \"Australian\"}", + 0, + 5.555555555555555 + ], + [ + 0.4444444444444444, + 5.485020637512207, + { + "__enum__": "StatusType.SUCCESS" + }, + 1637342743.4267018, + 1637342749.9442234, + { + "opt_loss": { + "accuracy": 0.4444444444444444, + "balanced_accuracy": 0.5, + "roc_auc": 0.5, + "average_precision": 0.5555555555555556, + "log_loss": 15.350567287868923, + "precision": 1.0, + "precision_macro": 0.7222222222222222, + "precision_micro": 0.4444444444444444, + "precision_weighted": 0.691358024691358, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.4444444444444444, + "recall_weighted": 0.4444444444444444, + "f1": 1.0, + "f1_macro": 0.6428571428571428, + "f1_micro": 0.4444444444444444, + "f1_weighted": 0.6031746031746031 + }, + "duration": 5.376826286315918, + "num_run": 16, + "train_loss": { + "accuracy": 0.45375722543352603, + "balanced_accuracy": 0.5, + "roc_auc": 0.5, + "average_precision": 0.546242774566474, + "log_loss": 15.67221934809161, + "precision": 1.0, + "precision_macro": 0.726878612716763, + "precision_micro": 0.45375722543352603, + "precision_weighted": 0.7016188312339203, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.45375722543352603, + "recall_weighted": 0.45375722543352603, + "f1": 1.0, + "f1_macro": 0.6467289719626168, + "f1_micro": 0.45375722543352603, + "f1_weighted": 0.6140565069418183 + }, + "test_loss": { + "accuracy": 0.4277456647398844, + "balanced_accuracy": 0.5, + "roc_auc": 0.5, + "average_precision": 0.5722543352601156, + "log_loss": 14.773811869538589, + "precision": 1.0, + "precision_macro": 0.7138728323699421, + "precision_micro": 0.4277456647398844, + "precision_weighted": 0.6725249757760032, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.4277456647398844, + "recall_weighted": 0.4277456647398844, + "f1": 1.0, + "f1_macro": 0.6360294117647058, + "f1_micro": 0.4277456647398844, + "f1_weighted": 0.5834325059503571 + }, + "configuration_origin": "Random Search" + } + ] + ], + [ + [ + 1, + "{\"task_id\": \"Australian\"}", + 0, + 16.666666666666664 + ], + [ + 0.15204678362573099, + 11.514830589294434, + { + "__enum__": "StatusType.SUCCESS" + }, + 1637342750.0053334, + 1637342762.5487585, + { + "opt_loss": { + "accuracy": 0.15204678362573099, + "balanced_accuracy": 0.15263157894736845, + "roc_auc": 0.08981994459833786, + "average_precision": 0.1040861796433199, + "log_loss": 0.5765479137672738, + "precision": 0.17948717948717952, + "precision_macro": 0.15425971877584788, + "precision_micro": 0.15204678362573099, + "precision_weighted": 0.15145666758569976, + "recall": 0.1578947368421053, + "recall_macro": 0.15263157894736845, + "recall_micro": 0.15204678362573099, + "recall_weighted": 0.15204678362573099, + "f1": 0.16883116883116878, + "f1_macro": 0.15356452058579717, + "f1_micro": 0.15204678362573099, + "f1_weighted": 0.15186822633631147 + }, + "duration": 11.44463586807251, + "num_run": 8, + "train_loss": { + "accuracy": 0.09537572254335258, + "balanced_accuracy": 0.10239948774980623, + "roc_auc": 0.03963198867657458, + "average_precision": 0.044469547423341305, + "log_loss": 0.28008669264774966, + "precision": 0.03731343283582089, + "precision_macro": 0.08469445226696704, + "precision_micro": 0.09537572254335258, + "precision_weighted": 0.0890765118675354, + "recall": 0.17834394904458595, + "recall_macro": 0.10239948774980623, + "recall_micro": 0.09537572254335258, + "recall_weighted": 0.09537572254335258, + "f1": 0.11340206185567014, + "f1_macro": 0.09784816309741107, + "f1_micro": 0.09537572254335258, + "f1_weighted": 0.0964096522295953 + }, + "test_loss": { + "accuracy": 0.1445086705202312, + "balanced_accuracy": 0.15356265356265353, + "roc_auc": 0.07821457821457822, + "average_precision": 0.08337624010373501, + "log_loss": 0.4291606295042948, + "precision": 0.13432835820895528, + "precision_macro": 0.14263587721768523, + "precision_micro": 0.1445086705202312, + "precision_weighted": 0.14383638574495827, + "recall": 0.21621621621621623, + "recall_macro": 0.15356265356265353, + "recall_micro": 0.1445086705202312, + "recall_weighted": 0.1445086705202312, + "f1": 0.17730496453900713, + "f1_macro": 0.14962809202560112, + "f1_micro": 0.1445086705202313, + "f1_weighted": 0.14562854397453084 + }, + "configuration_origin": "Default" + } + ] + ], + [ + [ + 1, + "{\"task_id\": \"Australian\"}", + 0, + 50.0 + ], + [ + 0.15204678362573099, + 15.370736837387085, + { + "__enum__": "StatusType.SUCCESS" + }, + 1637342762.794756, + 1637342779.192385, + { + "opt_loss": { + "accuracy": 0.15204678362573099, + "balanced_accuracy": 0.15263157894736845, + "roc_auc": 0.08981994459833786, + "average_precision": 0.1040861796433199, + "log_loss": 0.5765479137672738, + "precision": 0.17948717948717952, + "precision_macro": 0.15425971877584788, + "precision_micro": 0.15204678362573099, + "precision_weighted": 0.15145666758569976, + "recall": 0.1578947368421053, + "recall_macro": 0.15263157894736845, + "recall_micro": 0.15204678362573099, + "recall_weighted": 0.15204678362573099, + "f1": 0.16883116883116878, + "f1_macro": 0.15356452058579717, + "f1_micro": 0.15204678362573099, + "f1_weighted": 0.15186822633631147 + }, + "duration": 15.300711154937744, + "num_run": 8, + "train_loss": { + "accuracy": 0.09537572254335258, + "balanced_accuracy": 0.10239948774980623, + "roc_auc": 0.03963198867657458, + "average_precision": 0.044469547423341305, + "log_loss": 0.28008669264774966, + "precision": 0.03731343283582089, + "precision_macro": 0.08469445226696704, + "precision_micro": 0.09537572254335258, + "precision_weighted": 0.0890765118675354, + "recall": 0.17834394904458595, + "recall_macro": 0.10239948774980623, + "recall_micro": 0.09537572254335258, + "recall_weighted": 0.09537572254335258, + "f1": 0.11340206185567014, + "f1_macro": 0.09784816309741107, + "f1_micro": 0.09537572254335258, + "f1_weighted": 0.0964096522295953 + }, + "test_loss": { + "accuracy": 0.1445086705202312, + "balanced_accuracy": 0.15356265356265353, + "roc_auc": 0.07821457821457822, + "average_precision": 0.08337624010373501, + "log_loss": 0.4291606295042948, + "precision": 0.13432835820895528, + "precision_macro": 0.14263587721768523, + "precision_micro": 0.1445086705202312, + "precision_weighted": 0.14383638574495827, + "recall": 0.21621621621621623, + "recall_macro": 0.15356265356265353, + "recall_micro": 0.1445086705202312, + "recall_weighted": 0.1445086705202312, + "f1": 0.17730496453900713, + "f1_macro": 0.14962809202560112, + "f1_micro": 0.1445086705202313, + "f1_weighted": 0.14562854397453084 + }, + "configuration_origin": "Default" + } + ] + ], + [ + [ + 10, + "{\"task_id\": \"Australian\"}", + 0, + 16.666666666666664 + ], + [ + 0.4035087719298246, + 23.846530199050903, + { + "__enum__": "StatusType.SUCCESS" + }, + 1637342779.4572933, + 1637342804.3368232, + { + "opt_loss": { + "accuracy": 0.4035087719298246, + "balanced_accuracy": 0.39473684210526316, + "roc_auc": 0.3946675900277007, + "average_precision": 0.4846825737029168, + "log_loss": 6.419999084913276, + "precision": 0.4639175257731959, + "precision_macro": 0.39412092504876006, + "precision_micro": 0.4035087719298246, + "precision_weighted": 0.38636574719048944, + "recall": 0.3157894736842105, + "recall_macro": 0.39473684210526316, + "recall_micro": 0.4035087719298246, + "recall_weighted": 0.4035087719298246, + "f1": 0.3988439306358381, + "f1_macro": 0.4035639771522386, + "f1_micro": 0.4035087719298246, + "f1_weighted": 0.404088426765172 + }, + "duration": 23.588075160980225, + "num_run": 17, + "train_loss": { + "accuracy": 0.3988439306358381, + "balanced_accuracy": 0.3947359552455094, + "roc_auc": 0.4153776160145586, + "average_precision": 0.49525391358194226, + "log_loss": 7.800554750687936, + "precision": 0.4486486486486486, + "precision_macro": 0.39513177774047337, + "precision_micro": 0.3988439306358381, + "precision_weighted": 0.39018224054665374, + "recall": 0.35031847133757965, + "recall_macro": 0.3947359552455094, + "recall_micro": 0.3988439306358381, + "recall_weighted": 0.3988439306358381, + "f1": 0.4035087719298246, + "f1_macro": 0.39889724310776953, + "f1_micro": 0.3988439306358381, + "f1_weighted": 0.39847074333231924 + }, + "test_loss": { + "accuracy": 0.49132947976878616, + "balanced_accuracy": 0.4907179907179907, + "roc_auc": 0.45475020475020467, + "average_precision": 0.5080371591993963, + "log_loss": 8.656588848591884, + "precision": 0.5632183908045977, + "precision_macro": 0.49091152098369417, + "precision_micro": 0.49132947976878616, + "precision_weighted": 0.48046255135639593, + "recall": 0.4864864864864865, + "recall_macro": 0.4907179907179907, + "recall_micro": 0.49132947976878616, + "recall_weighted": 0.49132947976878616, + "f1": 0.5279503105590062, + "f1_macro": 0.4937048850092328, + "f1_micro": 0.49132947976878616, + "f1_weighted": 0.4887561240916355 + }, + "configuration_origin": "Random Search" + } + ] + ], + [ + [ + 11, + "{\"task_id\": \"Australian\"}", + 0, + 16.666666666666664 + ], + [ + 0.4444444444444444, + 6.757539510726929, + { + "__enum__": "StatusType.SUCCESS" + }, + 1637342813.02129, + 1637342820.8067145, + { + "opt_loss": { + "accuracy": 0.4444444444444444, + "balanced_accuracy": 0.5, + "roc_auc": 0.6250692520775624, + "average_precision": 0.642743659212315, + "log_loss": 0.6874508627674036, + "precision": 1.0, + "precision_macro": 0.7222222222222222, + "precision_micro": 0.4444444444444444, + "precision_weighted": 0.691358024691358, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.4444444444444444, + "recall_weighted": 0.4444444444444444, + "f1": 1.0, + "f1_macro": 0.6428571428571428, + "f1_micro": 0.4444444444444444, + "f1_weighted": 0.6031746031746031 + }, + "duration": 6.695321321487427, + "num_run": 18, + "train_loss": { + "accuracy": 0.45375722543352603, + "balanced_accuracy": 0.5, + "roc_auc": 0.5560273649445624, + "average_precision": 0.5899410773859022, + "log_loss": 0.689653262926664, + "precision": 1.0, + "precision_macro": 0.726878612716763, + "precision_micro": 0.45375722543352603, + "precision_weighted": 0.7016188312339203, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.45375722543352603, + "recall_weighted": 0.45375722543352603, + "f1": 1.0, + "f1_macro": 0.6467289719626168, + "f1_micro": 0.45375722543352603, + "f1_weighted": 0.6140565069418183 + }, + "test_loss": { + "accuracy": 0.4277456647398844, + "balanced_accuracy": 0.5, + "roc_auc": 0.558968058968059, + "average_precision": 0.6279915052238905, + "log_loss": 0.6829987437049777, + "precision": 1.0, + "precision_macro": 0.7138728323699421, + "precision_micro": 0.4277456647398844, + "precision_weighted": 0.6725249757760032, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.4277456647398844, + "recall_weighted": 0.4277456647398844, + "f1": 1.0, + "f1_macro": 0.6360294117647058, + "f1_micro": 0.4277456647398844, + "f1_weighted": 0.5834325059503571 + }, + "configuration_origin": "Random Search (sorted)" + } + ] + ], + [ + [ + 12, + "{\"task_id\": \"Australian\"}", + 0, + 16.666666666666664 + ], + [ + 0.4444444444444444, + 15.061991930007935, + { + "__enum__": "StatusType.SUCCESS" + }, + 1637342829.9214745, + 1637342846.0210106, + { + "opt_loss": { + "accuracy": 0.4444444444444444, + "balanced_accuracy": 0.5, + "roc_auc": 0.378393351800554, + "average_precision": 0.4680399341300143, + "log_loss": 0.6910723817278768, + "precision": 1.0, + "precision_macro": 0.7222222222222222, + "precision_micro": 0.4444444444444444, + "precision_weighted": 0.691358024691358, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.4444444444444444, + "recall_weighted": 0.4444444444444444, + "f1": 1.0, + "f1_macro": 0.6428571428571428, + "f1_micro": 0.4444444444444444, + "f1_weighted": 0.6031746031746031 + }, + "duration": 14.850486516952515, + "num_run": 19, + "train_loss": { + "accuracy": 0.45375722543352603, + "balanced_accuracy": 0.5, + "roc_auc": 0.44353452633707413, + "average_precision": 0.4796876232765652, + "log_loss": 0.6915661034556483, + "precision": 1.0, + "precision_macro": 0.726878612716763, + "precision_micro": 0.45375722543352603, + "precision_weighted": 0.7016188312339203, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.45375722543352603, + "recall_weighted": 0.45375722543352603, + "f1": 1.0, + "f1_macro": 0.6467289719626168, + "f1_micro": 0.45375722543352603, + "f1_weighted": 0.6140565069418183 + }, + "test_loss": { + "accuracy": 0.4277456647398844, + "balanced_accuracy": 0.5, + "roc_auc": 0.4836199836199836, + "average_precision": 0.5850466880722343, + "log_loss": 0.6912664895112803, + "precision": 1.0, + "precision_macro": 0.7138728323699421, + "precision_micro": 0.4277456647398844, + "precision_weighted": 0.6725249757760032, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.4277456647398844, + "recall_weighted": 0.4277456647398844, + "f1": 1.0, + "f1_macro": 0.6360294117647058, + "f1_micro": 0.4277456647398844, + "f1_weighted": 0.5834325059503571 + }, + "configuration_origin": "Random Search (sorted)" + } + ] + ], + [ + [ + 10, + "{\"task_id\": \"Australian\"}", + 0, + 50.0 + ], + [ + 1.0, + 50.010520696640015, + { + "__enum__": "StatusType.TIMEOUT" + }, + 1637342846.0745292, + 1637342897.1205413, + { + "error": "Timeout", + "configuration_origin": "Random Search" + } + ] + ], + [ + [ + 13, + "{\"task_id\": \"Australian\"}", + 0, + 50.0 + ], + [ + 1.0, + 22.011935234069824, + { + "__enum__": "StatusType.TIMEOUT" + }, + 1637342905.7068844, + 1637342928.7456856, + { + "error": "Timeout", + "configuration_origin": "Random Search (sorted)" + } + ] + ], + [ + [ + 14, + "{\"task_id\": \"Australian\"}", + 0, + 50.0 + ], + [ + 1.0, + 0.0, + { + "__enum__": "StatusType.STOP" + }, + 1637342928.8133125, + 1637342928.8133128, + {} + ] + ] + ], + "config_origins": { + "1": "Default", + "2": "Random Search", + "3": "Random Search (sorted)", + "4": "Random Search", + "5": "Random Search", + "6": "Random Search", + "7": "Random Search (sorted)", + "8": "Random Search (sorted)", + "9": "Random Search", + "10": "Random Search", + "11": "Random Search (sorted)", + "12": "Random Search (sorted)", + "13": "Random Search (sorted)", + "14": "Random Search" + }, + "configs": { + "1": { + "data_loader:batch_size": 64, + "encoder:__choice__": "OneHotEncoder", + "feature_preprocessor:__choice__": "NoFeaturePreprocessor", + "imputer:categorical_strategy": "most_frequent", + "imputer:numerical_strategy": "mean", + "lr_scheduler:__choice__": "ReduceLROnPlateau", + "network_backbone:__choice__": "ShapedMLPBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "XavierInit", + "optimizer:__choice__": "AdamOptimizer", + "scaler:__choice__": "StandardScaler", + "trainer:__choice__": "StandardTrainer", + "lr_scheduler:ReduceLROnPlateau:factor": 0.1, + "lr_scheduler:ReduceLROnPlateau:mode": "min", + "lr_scheduler:ReduceLROnPlateau:patience": 10, + "network_backbone:ShapedMLPBackbone:activation": "relu", + "network_backbone:ShapedMLPBackbone:max_units": 200, + "network_backbone:ShapedMLPBackbone:mlp_shape": "funnel", + "network_backbone:ShapedMLPBackbone:num_groups": 5, + "network_backbone:ShapedMLPBackbone:output_dim": 200, + "network_backbone:ShapedMLPBackbone:use_dropout": false, + "network_head:fully_connected:num_layers": 2, + "network_init:XavierInit:bias_strategy": "Normal", + "optimizer:AdamOptimizer:beta1": 0.9, + "optimizer:AdamOptimizer:beta2": 0.9, + "optimizer:AdamOptimizer:lr": 0.01, + "optimizer:AdamOptimizer:weight_decay": 0.0, + "trainer:StandardTrainer:weighted_loss": true, + "network_head:fully_connected:activation": "relu", + "network_head:fully_connected:units_layer_1": 128 + }, + "2": { + "data_loader:batch_size": 142, + "encoder:__choice__": "NoEncoder", + "feature_preprocessor:__choice__": "PowerTransformer", + "imputer:categorical_strategy": "constant_!missing!", + "imputer:numerical_strategy": "median", + "lr_scheduler:__choice__": "NoScheduler", + "network_backbone:__choice__": "ShapedResNetBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "KaimingInit", + "optimizer:__choice__": "AdamOptimizer", + "scaler:__choice__": "Normalizer", + "trainer:__choice__": "MixUpTrainer", + "feature_preprocessor:PowerTransformer:standardize": true, + "network_backbone:ShapedResNetBackbone:activation": "relu", + "network_backbone:ShapedResNetBackbone:blocks_per_group": 1, + "network_backbone:ShapedResNetBackbone:max_units": 175, + "network_backbone:ShapedResNetBackbone:num_groups": 3, + "network_backbone:ShapedResNetBackbone:output_dim": 550, + "network_backbone:ShapedResNetBackbone:resnet_shape": "funnel", + "network_backbone:ShapedResNetBackbone:use_dropout": false, + "network_backbone:ShapedResNetBackbone:use_shake_drop": true, + "network_backbone:ShapedResNetBackbone:use_shake_shake": true, + "network_head:fully_connected:num_layers": 2, + "network_init:KaimingInit:bias_strategy": "Normal", + "optimizer:AdamOptimizer:beta1": 0.8660298289969375, + "optimizer:AdamOptimizer:beta2": 0.9517157453274235, + "optimizer:AdamOptimizer:lr": 1.0377748473731365e-05, + "optimizer:AdamOptimizer:weight_decay": 0.07437634123996516, + "scaler:Normalizer:norm": "mean_abs", + "trainer:MixUpTrainer:alpha": 0.13179357367568267, + "trainer:MixUpTrainer:weighted_loss": true, + "network_backbone:ShapedResNetBackbone:max_shake_drop_probability": 0.7993610769045779, + "network_head:fully_connected:activation": "sigmoid", + "network_head:fully_connected:units_layer_1": 308 + }, + "3": { + "data_loader:batch_size": 246, + "encoder:__choice__": "OneHotEncoder", + "feature_preprocessor:__choice__": "PowerTransformer", + "imputer:categorical_strategy": "constant_!missing!", + "imputer:numerical_strategy": "most_frequent", + "lr_scheduler:__choice__": "CosineAnnealingWarmRestarts", + "network_backbone:__choice__": "ResNetBackbone", + "network_embedding:__choice__": "LearnedEntityEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "XavierInit", + "optimizer:__choice__": "AdamOptimizer", + "scaler:__choice__": "Normalizer", + "trainer:__choice__": "StandardTrainer", + "feature_preprocessor:PowerTransformer:standardize": true, + "lr_scheduler:CosineAnnealingWarmRestarts:T_0": 17, + "lr_scheduler:CosineAnnealingWarmRestarts:T_mult": 1.0577034671447638, + "network_backbone:ResNetBackbone:activation": "sigmoid", + "network_backbone:ResNetBackbone:blocks_per_group_0": 1, + "network_backbone:ResNetBackbone:blocks_per_group_1": 4, + "network_backbone:ResNetBackbone:num_groups": 10, + "network_backbone:ResNetBackbone:num_units_0": 974, + "network_backbone:ResNetBackbone:num_units_1": 151, + "network_backbone:ResNetBackbone:use_dropout": true, + "network_backbone:ResNetBackbone:use_shake_drop": true, + "network_backbone:ResNetBackbone:use_shake_shake": true, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_0": 0.6898659803969109, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_1": 0.6193894885012183, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_2": 0.27044405840757246, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_3": 0.3353276257116905, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_4": 0.25330009522745545, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_5": 0.28087428370045076, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_6": 0.985667346693578, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_7": 0.532995443030165, + "network_embedding:LearnedEntityEmbedding:min_unique_values_for_embedding": 4, + "network_head:fully_connected:num_layers": 4, + "network_init:XavierInit:bias_strategy": "Normal", + "optimizer:AdamOptimizer:beta1": 0.9319239463981089, + "optimizer:AdamOptimizer:beta2": 0.9290660642046109, + "optimizer:AdamOptimizer:lr": 4.934398361769327e-05, + "optimizer:AdamOptimizer:weight_decay": 0.0647885374302594, + "scaler:Normalizer:norm": "mean_abs", + "trainer:StandardTrainer:weighted_loss": true, + "network_backbone:ResNetBackbone:blocks_per_group_10": 2, + "network_backbone:ResNetBackbone:blocks_per_group_2": 1, + "network_backbone:ResNetBackbone:blocks_per_group_3": 1, + "network_backbone:ResNetBackbone:blocks_per_group_4": 3, + "network_backbone:ResNetBackbone:blocks_per_group_5": 1, + "network_backbone:ResNetBackbone:blocks_per_group_6": 3, + "network_backbone:ResNetBackbone:blocks_per_group_7": 4, + "network_backbone:ResNetBackbone:blocks_per_group_8": 1, + "network_backbone:ResNetBackbone:blocks_per_group_9": 3, + "network_backbone:ResNetBackbone:dropout_0": 0.1998483800982469, + "network_backbone:ResNetBackbone:dropout_1": 0.21671729531007777, + "network_backbone:ResNetBackbone:dropout_10": 0.2027668457966562, + "network_backbone:ResNetBackbone:dropout_2": 0.7140248388727144, + "network_backbone:ResNetBackbone:dropout_3": 0.1324478677866992, + "network_backbone:ResNetBackbone:dropout_4": 0.26711076053122573, + "network_backbone:ResNetBackbone:dropout_5": 0.2895993889716623, + "network_backbone:ResNetBackbone:dropout_6": 0.047419135928320616, + "network_backbone:ResNetBackbone:dropout_7": 0.593522761474697, + "network_backbone:ResNetBackbone:dropout_8": 0.11825542268484464, + "network_backbone:ResNetBackbone:dropout_9": 0.5802180655508312, + "network_backbone:ResNetBackbone:max_shake_drop_probability": 0.8422119101175598, + "network_backbone:ResNetBackbone:num_units_10": 1012, + "network_backbone:ResNetBackbone:num_units_2": 793, + "network_backbone:ResNetBackbone:num_units_3": 184, + "network_backbone:ResNetBackbone:num_units_4": 1022, + "network_backbone:ResNetBackbone:num_units_5": 88, + "network_backbone:ResNetBackbone:num_units_6": 666, + "network_backbone:ResNetBackbone:num_units_7": 927, + "network_backbone:ResNetBackbone:num_units_8": 614, + "network_backbone:ResNetBackbone:num_units_9": 552, + "network_head:fully_connected:activation": "relu", + "network_head:fully_connected:units_layer_1": 92, + "network_head:fully_connected:units_layer_2": 202, + "network_head:fully_connected:units_layer_3": 171 + }, + "4": { + "data_loader:batch_size": 269, + "encoder:__choice__": "OneHotEncoder", + "feature_preprocessor:__choice__": "PowerTransformer", + "imputer:categorical_strategy": "constant_!missing!", + "imputer:numerical_strategy": "median", + "lr_scheduler:__choice__": "CosineAnnealingLR", + "network_backbone:__choice__": "ShapedMLPBackbone", + "network_embedding:__choice__": "LearnedEntityEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "KaimingInit", + "optimizer:__choice__": "RMSpropOptimizer", + "scaler:__choice__": "MinMaxScaler", + "trainer:__choice__": "StandardTrainer", + "feature_preprocessor:PowerTransformer:standardize": true, + "lr_scheduler:CosineAnnealingLR:T_max": 57, + "network_backbone:ShapedMLPBackbone:activation": "relu", + "network_backbone:ShapedMLPBackbone:max_units": 199, + "network_backbone:ShapedMLPBackbone:mlp_shape": "stairs", + "network_backbone:ShapedMLPBackbone:num_groups": 12, + "network_backbone:ShapedMLPBackbone:output_dim": 641, + "network_backbone:ShapedMLPBackbone:use_dropout": true, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_0": 0.8093046015402414, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_1": 0.6952888698136637, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_2": 0.7136167420874352, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_3": 0.7071870846094686, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_4": 0.8821351885181623, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_5": 0.21840740866837938, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_6": 0.7366390825638998, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_7": 0.548715467945816, + "network_embedding:LearnedEntityEmbedding:min_unique_values_for_embedding": 7, + "network_head:fully_connected:num_layers": 3, + "network_init:KaimingInit:bias_strategy": "Zero", + "optimizer:RMSpropOptimizer:alpha": 0.23716801972855298, + "optimizer:RMSpropOptimizer:lr": 0.0011708542709120838, + "optimizer:RMSpropOptimizer:momentum": 0.5620565618493047, + "optimizer:RMSpropOptimizer:weight_decay": 0.05858239202799009, + "trainer:StandardTrainer:weighted_loss": true, + "network_backbone:ShapedMLPBackbone:max_dropout": 0.20819857031346878, + "network_head:fully_connected:activation": "relu", + "network_head:fully_connected:units_layer_1": 177, + "network_head:fully_connected:units_layer_2": 196 + }, + "5": { + "data_loader:batch_size": 191, + "encoder:__choice__": "OneHotEncoder", + "feature_preprocessor:__choice__": "RandomKitchenSinks", + "imputer:categorical_strategy": "constant_!missing!", + "imputer:numerical_strategy": "most_frequent", + "lr_scheduler:__choice__": "ExponentialLR", + "network_backbone:__choice__": "ShapedResNetBackbone", + "network_embedding:__choice__": "LearnedEntityEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "SparseInit", + "optimizer:__choice__": "AdamOptimizer", + "scaler:__choice__": "MinMaxScaler", + "trainer:__choice__": "MixUpTrainer", + "feature_preprocessor:RandomKitchenSinks:gamma": 0.00023806069646323692, + "feature_preprocessor:RandomKitchenSinks:n_components": 6, + "lr_scheduler:ExponentialLR:gamma": 0.7718494018636944, + "network_backbone:ShapedResNetBackbone:activation": "tanh", + "network_backbone:ShapedResNetBackbone:blocks_per_group": 3, + "network_backbone:ShapedResNetBackbone:max_units": 869, + "network_backbone:ShapedResNetBackbone:num_groups": 2, + "network_backbone:ShapedResNetBackbone:output_dim": 868, + "network_backbone:ShapedResNetBackbone:resnet_shape": "triangle", + "network_backbone:ShapedResNetBackbone:use_dropout": true, + "network_backbone:ShapedResNetBackbone:use_shake_drop": true, + "network_backbone:ShapedResNetBackbone:use_shake_shake": true, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_0": 0.08846693746970624, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_1": 0.6597252449477167, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_2": 0.11290616066859738, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_3": 0.4187266624427779, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_4": 0.026810815995375492, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_5": 0.02021466731982824, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_6": 0.01616376260397212, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_7": 0.6463510235731745, + "network_embedding:LearnedEntityEmbedding:min_unique_values_for_embedding": 6, + "network_head:fully_connected:num_layers": 3, + "network_init:SparseInit:bias_strategy": "Normal", + "optimizer:AdamOptimizer:beta1": 0.9966044931531224, + "optimizer:AdamOptimizer:beta2": 0.9293356180290759, + "optimizer:AdamOptimizer:lr": 0.07180366191531826, + "optimizer:AdamOptimizer:weight_decay": 0.012304534471441598, + "trainer:MixUpTrainer:alpha": 0.8900376828213522, + "trainer:MixUpTrainer:weighted_loss": true, + "network_backbone:ShapedResNetBackbone:max_dropout": 0.6688622458251051, + "network_backbone:ShapedResNetBackbone:max_shake_drop_probability": 0.28903761225065516, + "network_head:fully_connected:activation": "sigmoid", + "network_head:fully_connected:units_layer_1": 198, + "network_head:fully_connected:units_layer_2": 283 + }, + "6": { + "data_loader:batch_size": 53, + "encoder:__choice__": "OneHotEncoder", + "feature_preprocessor:__choice__": "PowerTransformer", + "imputer:categorical_strategy": "constant_!missing!", + "imputer:numerical_strategy": "median", + "lr_scheduler:__choice__": "StepLR", + "network_backbone:__choice__": "ResNetBackbone", + "network_embedding:__choice__": "LearnedEntityEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "OrthogonalInit", + "optimizer:__choice__": "AdamOptimizer", + "scaler:__choice__": "Normalizer", + "trainer:__choice__": "MixUpTrainer", + "feature_preprocessor:PowerTransformer:standardize": false, + "lr_scheduler:StepLR:gamma": 0.27529217359012764, + "lr_scheduler:StepLR:step_size": 8, + "network_backbone:ResNetBackbone:activation": "sigmoid", + "network_backbone:ResNetBackbone:blocks_per_group_0": 3, + "network_backbone:ResNetBackbone:blocks_per_group_1": 1, + "network_backbone:ResNetBackbone:num_groups": 6, + "network_backbone:ResNetBackbone:num_units_0": 884, + "network_backbone:ResNetBackbone:num_units_1": 160, + "network_backbone:ResNetBackbone:use_dropout": false, + "network_backbone:ResNetBackbone:use_shake_drop": true, + "network_backbone:ResNetBackbone:use_shake_shake": false, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_0": 0.5659324295712268, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_1": 0.8744001957677244, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_2": 0.3415903412295024, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_3": 0.8599314829187148, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_4": 0.9869678384973877, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_5": 0.7490528427155283, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_6": 0.9979477892240094, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_7": 0.4171316119626819, + "network_embedding:LearnedEntityEmbedding:min_unique_values_for_embedding": 5, + "network_head:fully_connected:num_layers": 1, + "network_init:OrthogonalInit:bias_strategy": "Zero", + "optimizer:AdamOptimizer:beta1": 0.9576776568384536, + "optimizer:AdamOptimizer:beta2": 0.9605074039230137, + "optimizer:AdamOptimizer:lr": 0.02098507521065345, + "optimizer:AdamOptimizer:weight_decay": 0.021686007599294888, + "scaler:Normalizer:norm": "mean_abs", + "trainer:MixUpTrainer:alpha": 0.8399712211486785, + "trainer:MixUpTrainer:weighted_loss": true, + "network_backbone:ResNetBackbone:blocks_per_group_2": 3, + "network_backbone:ResNetBackbone:blocks_per_group_3": 3, + "network_backbone:ResNetBackbone:blocks_per_group_4": 1, + "network_backbone:ResNetBackbone:blocks_per_group_5": 1, + "network_backbone:ResNetBackbone:blocks_per_group_6": 3, + "network_backbone:ResNetBackbone:max_shake_drop_probability": 0.09160627667494659, + "network_backbone:ResNetBackbone:num_units_2": 396, + "network_backbone:ResNetBackbone:num_units_3": 587, + "network_backbone:ResNetBackbone:num_units_4": 169, + "network_backbone:ResNetBackbone:num_units_5": 546, + "network_backbone:ResNetBackbone:num_units_6": 92 + }, + "7": { + "data_loader:batch_size": 232, + "encoder:__choice__": "NoEncoder", + "feature_preprocessor:__choice__": "RandomKitchenSinks", + "imputer:categorical_strategy": "most_frequent", + "imputer:numerical_strategy": "most_frequent", + "lr_scheduler:__choice__": "NoScheduler", + "network_backbone:__choice__": "ResNetBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "XavierInit", + "optimizer:__choice__": "AdamOptimizer", + "scaler:__choice__": "Normalizer", + "trainer:__choice__": "StandardTrainer", + "feature_preprocessor:RandomKitchenSinks:gamma": 5.7968061542283495e-05, + "feature_preprocessor:RandomKitchenSinks:n_components": 5, + "network_backbone:ResNetBackbone:activation": "sigmoid", + "network_backbone:ResNetBackbone:blocks_per_group_0": 2, + "network_backbone:ResNetBackbone:blocks_per_group_1": 3, + "network_backbone:ResNetBackbone:num_groups": 14, + "network_backbone:ResNetBackbone:num_units_0": 63, + "network_backbone:ResNetBackbone:num_units_1": 229, + "network_backbone:ResNetBackbone:use_dropout": true, + "network_backbone:ResNetBackbone:use_shake_drop": false, + "network_backbone:ResNetBackbone:use_shake_shake": true, + "network_head:fully_connected:num_layers": 2, + "network_init:XavierInit:bias_strategy": "Normal", + "optimizer:AdamOptimizer:beta1": 0.9646917651093316, + "optimizer:AdamOptimizer:beta2": 0.9949552394978046, + "optimizer:AdamOptimizer:lr": 0.018422761006289576, + "optimizer:AdamOptimizer:weight_decay": 0.01700341747601285, + "scaler:Normalizer:norm": "mean_squared", + "trainer:StandardTrainer:weighted_loss": false, + "network_backbone:ResNetBackbone:blocks_per_group_10": 4, + "network_backbone:ResNetBackbone:blocks_per_group_11": 3, + "network_backbone:ResNetBackbone:blocks_per_group_12": 2, + "network_backbone:ResNetBackbone:blocks_per_group_13": 1, + "network_backbone:ResNetBackbone:blocks_per_group_14": 3, + "network_backbone:ResNetBackbone:blocks_per_group_2": 1, + "network_backbone:ResNetBackbone:blocks_per_group_3": 3, + "network_backbone:ResNetBackbone:blocks_per_group_4": 4, + "network_backbone:ResNetBackbone:blocks_per_group_5": 3, + "network_backbone:ResNetBackbone:blocks_per_group_6": 4, + "network_backbone:ResNetBackbone:blocks_per_group_7": 2, + "network_backbone:ResNetBackbone:blocks_per_group_8": 3, + "network_backbone:ResNetBackbone:blocks_per_group_9": 2, + "network_backbone:ResNetBackbone:dropout_0": 0.3872694110962167, + "network_backbone:ResNetBackbone:dropout_1": 0.7182095865182352, + "network_backbone:ResNetBackbone:dropout_10": 0.7518775284870586, + "network_backbone:ResNetBackbone:dropout_11": 0.3717581189860213, + "network_backbone:ResNetBackbone:dropout_12": 0.055178370982331075, + "network_backbone:ResNetBackbone:dropout_13": 0.5670307132839905, + "network_backbone:ResNetBackbone:dropout_14": 0.7859566818562779, + "network_backbone:ResNetBackbone:dropout_2": 0.5796670187291707, + "network_backbone:ResNetBackbone:dropout_3": 0.05370643307213783, + "network_backbone:ResNetBackbone:dropout_4": 0.37288408223729974, + "network_backbone:ResNetBackbone:dropout_5": 0.47179695650262793, + "network_backbone:ResNetBackbone:dropout_6": 0.20070003914010803, + "network_backbone:ResNetBackbone:dropout_7": 0.638048407623313, + "network_backbone:ResNetBackbone:dropout_8": 0.6190670404279601, + "network_backbone:ResNetBackbone:dropout_9": 0.33325853682297146, + "network_backbone:ResNetBackbone:num_units_10": 925, + "network_backbone:ResNetBackbone:num_units_11": 164, + "network_backbone:ResNetBackbone:num_units_12": 247, + "network_backbone:ResNetBackbone:num_units_13": 339, + "network_backbone:ResNetBackbone:num_units_14": 769, + "network_backbone:ResNetBackbone:num_units_2": 502, + "network_backbone:ResNetBackbone:num_units_3": 101, + "network_backbone:ResNetBackbone:num_units_4": 842, + "network_backbone:ResNetBackbone:num_units_5": 906, + "network_backbone:ResNetBackbone:num_units_6": 933, + "network_backbone:ResNetBackbone:num_units_7": 329, + "network_backbone:ResNetBackbone:num_units_8": 898, + "network_backbone:ResNetBackbone:num_units_9": 161, + "network_head:fully_connected:activation": "relu", + "network_head:fully_connected:units_layer_1": 327 + }, + "8": { + "data_loader:batch_size": 164, + "encoder:__choice__": "NoEncoder", + "feature_preprocessor:__choice__": "NoFeaturePreprocessor", + "imputer:categorical_strategy": "most_frequent", + "imputer:numerical_strategy": "mean", + "lr_scheduler:__choice__": "StepLR", + "network_backbone:__choice__": "ShapedMLPBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "NoInit", + "optimizer:__choice__": "RMSpropOptimizer", + "scaler:__choice__": "MinMaxScaler", + "trainer:__choice__": "MixUpTrainer", + "lr_scheduler:StepLR:gamma": 0.2905213739360219, + "lr_scheduler:StepLR:step_size": 10, + "network_backbone:ShapedMLPBackbone:activation": "sigmoid", + "network_backbone:ShapedMLPBackbone:max_units": 903, + "network_backbone:ShapedMLPBackbone:mlp_shape": "brick", + "network_backbone:ShapedMLPBackbone:num_groups": 10, + "network_backbone:ShapedMLPBackbone:output_dim": 943, + "network_backbone:ShapedMLPBackbone:use_dropout": false, + "network_head:fully_connected:num_layers": 3, + "network_init:NoInit:bias_strategy": "Zero", + "optimizer:RMSpropOptimizer:alpha": 0.25445785033325663, + "optimizer:RMSpropOptimizer:lr": 0.00012058949092384073, + "optimizer:RMSpropOptimizer:momentum": 0.6601732030357997, + "optimizer:RMSpropOptimizer:weight_decay": 0.030275825765581223, + "trainer:MixUpTrainer:alpha": 0.2222082093355312, + "trainer:MixUpTrainer:weighted_loss": true, + "network_head:fully_connected:activation": "sigmoid", + "network_head:fully_connected:units_layer_1": 110, + "network_head:fully_connected:units_layer_2": 70 + }, + "9": { + "data_loader:batch_size": 94, + "encoder:__choice__": "NoEncoder", + "feature_preprocessor:__choice__": "PolynomialFeatures", + "imputer:categorical_strategy": "most_frequent", + "imputer:numerical_strategy": "mean", + "lr_scheduler:__choice__": "CyclicLR", + "network_backbone:__choice__": "MLPBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "KaimingInit", + "optimizer:__choice__": "RMSpropOptimizer", + "scaler:__choice__": "StandardScaler", + "trainer:__choice__": "StandardTrainer", + "feature_preprocessor:PolynomialFeatures:degree": 2, + "feature_preprocessor:PolynomialFeatures:include_bias": false, + "feature_preprocessor:PolynomialFeatures:interaction_only": false, + "lr_scheduler:CyclicLR:base_lr": 0.09510483864725039, + "lr_scheduler:CyclicLR:max_lr": 0.026215723559513626, + "lr_scheduler:CyclicLR:mode": "triangular", + "lr_scheduler:CyclicLR:step_size_up": 3829, + "network_backbone:MLPBackbone:activation": "sigmoid", + "network_backbone:MLPBackbone:num_groups": 7, + "network_backbone:MLPBackbone:num_units_1": 47, + "network_backbone:MLPBackbone:use_dropout": true, + "network_head:fully_connected:num_layers": 3, + "network_init:KaimingInit:bias_strategy": "Zero", + "optimizer:RMSpropOptimizer:alpha": 0.75085811094601, + "optimizer:RMSpropOptimizer:lr": 0.0002950013672615944, + "optimizer:RMSpropOptimizer:momentum": 0.515129966307681, + "optimizer:RMSpropOptimizer:weight_decay": 0.01979731884468683, + "trainer:StandardTrainer:weighted_loss": true, + "network_backbone:MLPBackbone:dropout_1": 0.5362963908147109, + "network_backbone:MLPBackbone:dropout_2": 0.09403575191589564, + "network_backbone:MLPBackbone:dropout_3": 0.5576340928985162, + "network_backbone:MLPBackbone:dropout_4": 0.3102921398336836, + "network_backbone:MLPBackbone:dropout_5": 0.36841155269138837, + "network_backbone:MLPBackbone:dropout_6": 0.459182557172949, + "network_backbone:MLPBackbone:dropout_7": 0.2741849570242409, + "network_backbone:MLPBackbone:num_units_2": 323, + "network_backbone:MLPBackbone:num_units_3": 424, + "network_backbone:MLPBackbone:num_units_4": 637, + "network_backbone:MLPBackbone:num_units_5": 668, + "network_backbone:MLPBackbone:num_units_6": 507, + "network_backbone:MLPBackbone:num_units_7": 972, + "network_head:fully_connected:activation": "sigmoid", + "network_head:fully_connected:units_layer_1": 482, + "network_head:fully_connected:units_layer_2": 425 + }, + "10": { + "data_loader:batch_size": 70, + "encoder:__choice__": "OneHotEncoder", + "feature_preprocessor:__choice__": "PowerTransformer", + "imputer:categorical_strategy": "most_frequent", + "imputer:numerical_strategy": "constant_zero", + "lr_scheduler:__choice__": "CyclicLR", + "network_backbone:__choice__": "ShapedMLPBackbone", + "network_embedding:__choice__": "LearnedEntityEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "SparseInit", + "optimizer:__choice__": "RMSpropOptimizer", + "scaler:__choice__": "NoScaler", + "trainer:__choice__": "MixUpTrainer", + "feature_preprocessor:PowerTransformer:standardize": false, + "lr_scheduler:CyclicLR:base_lr": 0.05611929109855669, + "lr_scheduler:CyclicLR:max_lr": 0.01831055731936772, + "lr_scheduler:CyclicLR:mode": "triangular2", + "lr_scheduler:CyclicLR:step_size_up": 3104, + "network_backbone:ShapedMLPBackbone:activation": "tanh", + "network_backbone:ShapedMLPBackbone:max_units": 759, + "network_backbone:ShapedMLPBackbone:mlp_shape": "brick", + "network_backbone:ShapedMLPBackbone:num_groups": 10, + "network_backbone:ShapedMLPBackbone:output_dim": 155, + "network_backbone:ShapedMLPBackbone:use_dropout": true, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_0": 0.14333490052026576, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_1": 0.7794060644969828, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_2": 0.28020395999441905, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_3": 0.2820327943739419, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_4": 0.7390548552027222, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_5": 0.025302711343403672, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_6": 0.5677825375428477, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_7": 0.7093786601691139, + "network_embedding:LearnedEntityEmbedding:min_unique_values_for_embedding": 6, + "network_head:fully_connected:num_layers": 4, + "network_init:SparseInit:bias_strategy": "Zero", + "optimizer:RMSpropOptimizer:alpha": 0.15659144532965727, + "optimizer:RMSpropOptimizer:lr": 0.015691888676781927, + "optimizer:RMSpropOptimizer:momentum": 0.30317416976729206, + "optimizer:RMSpropOptimizer:weight_decay": 0.010642526008626797, + "trainer:MixUpTrainer:alpha": 0.3709089665342886, + "trainer:MixUpTrainer:weighted_loss": false, + "network_backbone:ShapedMLPBackbone:max_dropout": 0.3789762581174825, + "network_head:fully_connected:activation": "tanh", + "network_head:fully_connected:units_layer_1": 499, + "network_head:fully_connected:units_layer_2": 465, + "network_head:fully_connected:units_layer_3": 238 + }, + "11": { + "data_loader:batch_size": 274, + "encoder:__choice__": "NoEncoder", + "feature_preprocessor:__choice__": "RandomKitchenSinks", + "imputer:categorical_strategy": "constant_!missing!", + "imputer:numerical_strategy": "mean", + "lr_scheduler:__choice__": "CyclicLR", + "network_backbone:__choice__": "ShapedMLPBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "SparseInit", + "optimizer:__choice__": "AdamOptimizer", + "scaler:__choice__": "Normalizer", + "trainer:__choice__": "StandardTrainer", + "feature_preprocessor:RandomKitchenSinks:gamma": 0.00017836687895829377, + "feature_preprocessor:RandomKitchenSinks:n_components": 3, + "lr_scheduler:CyclicLR:base_lr": 0.061001847805883254, + "lr_scheduler:CyclicLR:max_lr": 0.037867703829357294, + "lr_scheduler:CyclicLR:mode": "triangular", + "lr_scheduler:CyclicLR:step_size_up": 3395, + "network_backbone:ShapedMLPBackbone:activation": "relu", + "network_backbone:ShapedMLPBackbone:max_units": 94, + "network_backbone:ShapedMLPBackbone:mlp_shape": "diamond", + "network_backbone:ShapedMLPBackbone:num_groups": 4, + "network_backbone:ShapedMLPBackbone:output_dim": 763, + "network_backbone:ShapedMLPBackbone:use_dropout": false, + "network_head:fully_connected:num_layers": 3, + "network_init:SparseInit:bias_strategy": "Zero", + "optimizer:AdamOptimizer:beta1": 0.9010241766841086, + "optimizer:AdamOptimizer:beta2": 0.9275862063741073, + "optimizer:AdamOptimizer:lr": 0.00048241454070108375, + "optimizer:AdamOptimizer:weight_decay": 0.058438892093437125, + "scaler:Normalizer:norm": "max", + "trainer:StandardTrainer:weighted_loss": true, + "network_head:fully_connected:activation": "relu", + "network_head:fully_connected:units_layer_1": 293, + "network_head:fully_connected:units_layer_2": 177 + }, + "12": { + "data_loader:batch_size": 191, + "encoder:__choice__": "NoEncoder", + "feature_preprocessor:__choice__": "NoFeaturePreprocessor", + "imputer:categorical_strategy": "constant_!missing!", + "imputer:numerical_strategy": "median", + "lr_scheduler:__choice__": "CosineAnnealingWarmRestarts", + "network_backbone:__choice__": "ResNetBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "XavierInit", + "optimizer:__choice__": "RMSpropOptimizer", + "scaler:__choice__": "Normalizer", + "trainer:__choice__": "StandardTrainer", + "lr_scheduler:CosineAnnealingWarmRestarts:T_0": 18, + "lr_scheduler:CosineAnnealingWarmRestarts:T_mult": 1.7405132785152093, + "network_backbone:ResNetBackbone:activation": "relu", + "network_backbone:ResNetBackbone:blocks_per_group_0": 1, + "network_backbone:ResNetBackbone:blocks_per_group_1": 4, + "network_backbone:ResNetBackbone:num_groups": 6, + "network_backbone:ResNetBackbone:num_units_0": 894, + "network_backbone:ResNetBackbone:num_units_1": 395, + "network_backbone:ResNetBackbone:use_dropout": true, + "network_backbone:ResNetBackbone:use_shake_drop": false, + "network_backbone:ResNetBackbone:use_shake_shake": false, + "network_head:fully_connected:num_layers": 4, + "network_init:XavierInit:bias_strategy": "Zero", + "optimizer:RMSpropOptimizer:alpha": 0.6521242194975473, + "optimizer:RMSpropOptimizer:lr": 4.097035283946373e-05, + "optimizer:RMSpropOptimizer:momentum": 0.1792833337110808, + "optimizer:RMSpropOptimizer:weight_decay": 0.006909623450893943, + "scaler:Normalizer:norm": "mean_abs", + "trainer:StandardTrainer:weighted_loss": false, + "network_backbone:ResNetBackbone:blocks_per_group_2": 2, + "network_backbone:ResNetBackbone:blocks_per_group_3": 3, + "network_backbone:ResNetBackbone:blocks_per_group_4": 2, + "network_backbone:ResNetBackbone:blocks_per_group_5": 4, + "network_backbone:ResNetBackbone:blocks_per_group_6": 1, + "network_backbone:ResNetBackbone:dropout_0": 0.6575114752945207, + "network_backbone:ResNetBackbone:dropout_1": 0.28916184819601504, + "network_backbone:ResNetBackbone:dropout_2": 0.09888388652277876, + "network_backbone:ResNetBackbone:dropout_3": 0.791809735686961, + "network_backbone:ResNetBackbone:dropout_4": 0.06432675017963892, + "network_backbone:ResNetBackbone:dropout_5": 0.3015819044494064, + "network_backbone:ResNetBackbone:dropout_6": 0.792332044450592, + "network_backbone:ResNetBackbone:num_units_2": 173, + "network_backbone:ResNetBackbone:num_units_3": 290, + "network_backbone:ResNetBackbone:num_units_4": 633, + "network_backbone:ResNetBackbone:num_units_5": 16, + "network_backbone:ResNetBackbone:num_units_6": 542, + "network_head:fully_connected:activation": "relu", + "network_head:fully_connected:units_layer_1": 429, + "network_head:fully_connected:units_layer_2": 342, + "network_head:fully_connected:units_layer_3": 322 + }, + "13": { + "data_loader:batch_size": 35, + "encoder:__choice__": "OneHotEncoder", + "feature_preprocessor:__choice__": "PowerTransformer", + "imputer:categorical_strategy": "most_frequent", + "imputer:numerical_strategy": "most_frequent", + "lr_scheduler:__choice__": "ExponentialLR", + "network_backbone:__choice__": "ShapedMLPBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "XavierInit", + "optimizer:__choice__": "AdamWOptimizer", + "scaler:__choice__": "Normalizer", + "trainer:__choice__": "MixUpTrainer", + "feature_preprocessor:PowerTransformer:standardize": false, + "lr_scheduler:ExponentialLR:gamma": 0.863670772292724, + "network_backbone:ShapedMLPBackbone:activation": "sigmoid", + "network_backbone:ShapedMLPBackbone:max_units": 957, + "network_backbone:ShapedMLPBackbone:mlp_shape": "long_funnel", + "network_backbone:ShapedMLPBackbone:num_groups": 7, + "network_backbone:ShapedMLPBackbone:output_dim": 16, + "network_backbone:ShapedMLPBackbone:use_dropout": true, + "network_head:fully_connected:num_layers": 3, + "network_init:XavierInit:bias_strategy": "Normal", + "optimizer:AdamWOptimizer:beta1": 0.9298951109018316, + "optimizer:AdamWOptimizer:beta2": 0.9367719861032991, + "optimizer:AdamWOptimizer:lr": 2.3043911799203502e-05, + "optimizer:AdamWOptimizer:weight_decay": 0.08948752020001628, + "scaler:Normalizer:norm": "max", + "trainer:MixUpTrainer:alpha": 0.1848582510096881, + "trainer:MixUpTrainer:weighted_loss": true, + "network_backbone:ShapedMLPBackbone:max_dropout": 0.4933977554884884, + "network_head:fully_connected:activation": "relu", + "network_head:fully_connected:units_layer_1": 105, + "network_head:fully_connected:units_layer_2": 185 + }, + "14": { + "data_loader:batch_size": 154, + "encoder:__choice__": "OneHotEncoder", + "feature_preprocessor:__choice__": "KernelPCA", + "imputer:categorical_strategy": "most_frequent", + "imputer:numerical_strategy": "mean", + "lr_scheduler:__choice__": "StepLR", + "network_backbone:__choice__": "ResNetBackbone", + "network_embedding:__choice__": "LearnedEntityEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "NoInit", + "optimizer:__choice__": "AdamWOptimizer", + "scaler:__choice__": "NoScaler", + "trainer:__choice__": "StandardTrainer", + "feature_preprocessor:KernelPCA:kernel": "sigmoid", + "feature_preprocessor:KernelPCA:n_components": 3, + "lr_scheduler:StepLR:gamma": 0.5658285105415104, + "lr_scheduler:StepLR:step_size": 10, + "network_backbone:ResNetBackbone:activation": "tanh", + "network_backbone:ResNetBackbone:blocks_per_group_0": 2, + "network_backbone:ResNetBackbone:blocks_per_group_1": 2, + "network_backbone:ResNetBackbone:num_groups": 1, + "network_backbone:ResNetBackbone:num_units_0": 623, + "network_backbone:ResNetBackbone:num_units_1": 42, + "network_backbone:ResNetBackbone:use_dropout": false, + "network_backbone:ResNetBackbone:use_shake_drop": true, + "network_backbone:ResNetBackbone:use_shake_shake": true, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_0": 0.7061800992159439, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_1": 0.40404533505032336, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_2": 0.14124612419045746, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_3": 0.24304972767199295, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_4": 0.8403938666630251, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_5": 0.11081539209354929, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_6": 0.5150164644256714, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_7": 0.6185258490472787, + "network_embedding:LearnedEntityEmbedding:min_unique_values_for_embedding": 3, + "network_head:fully_connected:num_layers": 2, + "network_init:NoInit:bias_strategy": "Zero", + "optimizer:AdamWOptimizer:beta1": 0.9639206805787317, + "optimizer:AdamWOptimizer:beta2": 0.9439342949959634, + "optimizer:AdamWOptimizer:lr": 0.05110804312778185, + "optimizer:AdamWOptimizer:weight_decay": 0.026136253949706992, + "trainer:StandardTrainer:weighted_loss": true, + "feature_preprocessor:KernelPCA:coef0": 0.27733876378393374, + "network_backbone:ResNetBackbone:max_shake_drop_probability": 0.4280891218905112, + "network_head:fully_connected:activation": "relu", + "network_head:fully_connected:units_layer_1": 506 + } + } +} \ No newline at end of file diff --git a/test/test_api/test_api.py b/test/test_api/test_api.py index 5f670e59d..5cb271eb0 100644 --- a/test/test_api/test_api.py +++ b/test/test_api/test_api.py @@ -3,7 +3,7 @@ import pathlib import pickle import unittest -from test.test_api.utils import dummy_do_dummy_prediction, dummy_eval_function, dummy_traditional_classification +from test.test_api.utils import dummy_do_dummy_prediction, dummy_eval_function import ConfigSpace as CS from ConfigSpace.configuration_space import Configuration @@ -25,12 +25,10 @@ from autoPyTorch.api.tabular_classification import TabularClassificationTask from autoPyTorch.api.tabular_regression import TabularRegressionTask -from autoPyTorch.data.tabular_validator import TabularInputValidator from autoPyTorch.datasets.resampling_strategy import ( CrossValTypes, HoldoutValTypes, ) -from autoPyTorch.datasets.tabular_dataset import TabularDataset from autoPyTorch.optimizer.smbo import AutoMLSMBO from autoPyTorch.pipeline.base_pipeline import BasePipeline from autoPyTorch.pipeline.components.setup.traditional_ml.traditional_learner import _traditional_learners @@ -575,73 +573,6 @@ def test_portfolio_selection_failure(openml_id, backend, n_samples): ) -@pytest.mark.parametrize('dataset_name', ('iris',)) -@pytest.mark.parametrize('include_traditional', (True, False)) -def test_get_incumbent_results(dataset_name, backend, include_traditional): - # Get the data and check that contents of data-manager make sense - X, y = sklearn.datasets.fetch_openml( - name=dataset_name, - return_X_y=True, as_frame=True - ) - - X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split( - X, y, random_state=1) - - # Search for a good configuration - estimator = TabularClassificationTask( - backend=backend, - resampling_strategy=HoldoutValTypes.holdout_validation, - ) - - InputValidator = TabularInputValidator( - is_classification=True, - ) - - # Fit a input validator to check the provided data - # Also, an encoder is fit to both train and test data, - # to prevent unseen categories during inference - InputValidator.fit(X_train=X_train, y_train=y_train, X_test=X_test, y_test=y_test) - - dataset = TabularDataset( - X=X_train, Y=y_train, - X_test=X_test, Y_test=y_test, - validator=InputValidator, - resampling_strategy=estimator.resampling_strategy, - resampling_strategy_args=estimator.resampling_strategy_args, - ) - - pipeline_run_history = RunHistory() - pipeline_run_history.load_json(os.path.join(os.path.dirname(__file__), '.tmp_api/runhistory.json'), - estimator.get_search_space(dataset)) - - estimator._do_dummy_prediction = unittest.mock.MagicMock() - - with unittest.mock.patch.object(AutoMLSMBO, 'run_smbo') as AutoMLSMBOMock: - with unittest.mock.patch.object(TabularClassificationTask, '_do_traditional_prediction', - new=dummy_traditional_classification): - AutoMLSMBOMock.return_value = (pipeline_run_history, {}, 'epochs') - estimator.search( - X_train=X_train, y_train=y_train, - X_test=X_test, y_test=y_test, - optimize_metric='accuracy', - total_walltime_limit=150, - func_eval_time_limit_secs=50, - enable_traditional_pipeline=True, - load_models=False, - ) - config, results = estimator.get_incumbent_results(include_traditional=include_traditional) - assert isinstance(config, Configuration) - assert isinstance(results, dict) - - run_history_data = estimator.run_history.data - costs = [run_value.cost for run_key, run_value in run_history_data.items() if run_value.additional_info is not None - and (run_value.additional_info['configuration_origin'] != 'traditional' or include_traditional)] - assert results['opt_loss']['accuracy'] == min(costs) - - if not include_traditional: - assert results['configuration_origin'] != 'traditional' - - # TODO: Make faster when https://github.com/automl/Auto-PyTorch/pull/223 is incorporated @pytest.mark.parametrize("fit_dictionary_tabular", ['classification_categorical_only'], indirect=True) def test_do_traditional_pipeline(fit_dictionary_tabular): diff --git a/test/test_api/test_results_manager.py b/test/test_api/test_results_manager.py new file mode 100644 index 000000000..4c6e7a7ae --- /dev/null +++ b/test/test_api/test_results_manager.py @@ -0,0 +1,232 @@ +import json +import os +from test.test_api.utils import make_dict_run_history_data +from unittest.mock import MagicMock + +import ConfigSpace.hyperparameters as CSH +from ConfigSpace.configuration_space import Configuration, ConfigurationSpace + +import numpy as np + +import pytest + +from smac.runhistory.runhistory import RunHistory, StatusType + +from autoPyTorch.api.base_task import BaseTask +from autoPyTorch.api.results_manager import ResultsManager, STATUS2MSG, SearchResults, cost2metric +from autoPyTorch.metrics import accuracy, balanced_accuracy, log_loss + + +def _check_status(status): + """ Based on runhistory_B.json """ + ans = [ + STATUS2MSG[StatusType.SUCCESS], STATUS2MSG[StatusType.SUCCESS], + STATUS2MSG[StatusType.SUCCESS], STATUS2MSG[StatusType.SUCCESS], + STATUS2MSG[StatusType.SUCCESS], STATUS2MSG[StatusType.SUCCESS], + STATUS2MSG[StatusType.CRASHED], STATUS2MSG[StatusType.SUCCESS], + STATUS2MSG[StatusType.SUCCESS], STATUS2MSG[StatusType.SUCCESS], + STATUS2MSG[StatusType.SUCCESS], STATUS2MSG[StatusType.SUCCESS], + STATUS2MSG[StatusType.SUCCESS], STATUS2MSG[StatusType.SUCCESS], + STATUS2MSG[StatusType.TIMEOUT], STATUS2MSG[StatusType.TIMEOUT], + ] + assert isinstance(status, list) + assert isinstance(status[0], str) + assert status == ans + + +def _check_costs(costs): + """ Based on runhistory_B.json """ + ans = [0.15204678362573099, 0.4444444444444444, 0.5555555555555556, 0.29824561403508776, + 0.4444444444444444, 0.4444444444444444, 1.0, 0.5555555555555556, 0.4444444444444444, + 0.15204678362573099, 0.15204678362573099, 0.4035087719298246, 0.4444444444444444, + 0.4444444444444444, 1.0, 1.0] + assert np.allclose(1 - np.array(costs), ans) + assert isinstance(costs, np.ndarray) + assert costs.dtype is np.dtype(np.float) + + +def _check_fit_times(fit_times): + """ Based on runhistory_B.json """ + ans = [3.154788017272949, 3.2763524055480957, 22.723600149154663, 4.990685224533081, 10.684926509857178, + 9.947429180145264, 11.687273979187012, 8.478890419006348, 5.485020637512207, 11.514830589294434, + 15.370736837387085, 23.846530199050903, 6.757539510726929, 15.061991930007935, 50.010520696640015, + 22.011935234069824] + + assert np.allclose(fit_times, ans) + assert isinstance(fit_times, np.ndarray) + assert fit_times.dtype is np.dtype(np.float) + + +def _check_budgets(budgets): + """ Based on runhistory_B.json """ + ans = [5.555555555555555, 5.555555555555555, 5.555555555555555, 5.555555555555555, + 5.555555555555555, 5.555555555555555, 5.555555555555555, 5.555555555555555, + 5.555555555555555, 16.666666666666664, 50.0, 16.666666666666664, 16.666666666666664, + 16.666666666666664, 50.0, 50.0] + assert np.allclose(budgets, ans) + assert isinstance(budgets, list) + assert isinstance(budgets[0], float) + + +def _check_additional_infos(status_types, additional_infos): + for i, status in enumerate(status_types): + info = additional_infos[i] + if status in (STATUS2MSG[StatusType.SUCCESS], STATUS2MSG[StatusType.DONOTADVANCE]): + metric_info = info.get('opt_loss', None) + assert metric_info is not None + elif info is not None: + metric_info = info.get('opt_loss', None) + assert metric_info is None + + +def _check_metric_dict(metric_dict, status_types): + assert isinstance(metric_dict['accuracy'], list) + assert metric_dict['accuracy'][0] > 0 + assert isinstance(metric_dict['balanced_accuracy'], list) + assert metric_dict['balanced_accuracy'][0] > 0 + + for key, vals in metric_dict.items(): + # ^ is a XOR operator + # True and False / False and True must be fulfilled + assert all([(s == STATUS2MSG[StatusType.SUCCESS]) ^ isnan + for s, isnan in zip(status_types, np.isnan(vals))]) + + +def test_extract_results_from_run_history(): + # test the raise error for the `status_msg is None` + run_history = RunHistory() + cs = ConfigurationSpace() + config = Configuration(cs, {}) + run_history.add( + config=config, + cost=0.0, + time=1.0, + status=StatusType.CAPPED, + ) + with pytest.raises(ValueError) as excinfo: + SearchResults(metric=accuracy, scoring_functions=[], run_history=run_history) + + assert excinfo._excinfo[0] == ValueError + + +def test_search_results_sprint_statistics(): + api = BaseTask() + for method in ['get_search_results', 'sprint_statistics', 'get_incumbent_results']: + with pytest.raises(RuntimeError) as excinfo: + getattr(api, method)() + + assert excinfo._excinfo[0] == RuntimeError + + run_history_data = json.load(open(os.path.join(os.path.dirname(__file__), + '.tmp_api/runhistory_B.json'), + mode='r'))['data'] + api._results_manager.run_history = MagicMock() + api.run_history.empty = MagicMock(return_value=False) + + # The run_history has 16 runs + 1 run interruption ==> 16 runs + api.run_history.data = make_dict_run_history_data(run_history_data) + api._metric = accuracy + api.dataset_name = 'iris' + api._scoring_functions = [accuracy, balanced_accuracy] + api.search_space = MagicMock(spec=ConfigurationSpace) + search_results = api.get_search_results() + + _check_status(search_results.status_types) + _check_costs(search_results.opt_scores) + _check_fit_times(search_results.fit_times) + _check_budgets(search_results.budgets) + _check_metric_dict(search_results.metric_dict, search_results.status_types) + _check_additional_infos(status_types=search_results.status_types, + additional_infos=search_results.additional_infos) + + # config_ids can duplicate because of various budget size + config_ids = [1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 1, 10, 11, 12, 10, 13] + assert config_ids == search_results.config_ids + + # assert that contents of search_results are of expected types + assert isinstance(search_results.rank_test_scores, np.ndarray) + assert search_results.rank_test_scores.dtype is np.dtype(np.int) + assert isinstance(search_results.configs, list) + + n_success, n_timeout, n_memoryout, n_crashed = 13, 2, 0, 1 + msg = ["autoPyTorch results:", f"\tDataset name: {api.dataset_name}", + f"\tOptimisation Metric: {api._metric.name}", + f"\tBest validation score: {max(search_results.opt_scores)}", + "\tNumber of target algorithm runs: 16", f"\tNumber of successful target algorithm runs: {n_success}", + f"\tNumber of crashed target algorithm runs: {n_crashed}", + f"\tNumber of target algorithms that exceeded the time limit: {n_timeout}", + f"\tNumber of target algorithms that exceeded the memory limit: {n_memoryout}"] + + assert isinstance(api.sprint_statistics(), str) + assert all([m1 == m2 for m1, m2 in zip(api.sprint_statistics().split("\n"), msg)]) + + +@pytest.mark.parametrize('run_history', (None, RunHistory())) +def test_check_run_history(run_history): + manager = ResultsManager() + manager.run_history = run_history + + with pytest.raises(RuntimeError) as excinfo: + manager._check_run_history() + + assert excinfo._excinfo[0] == RuntimeError + + +T, NT = 'traditional', 'non-traditional' +SCORES = [0.1 * (i + 1) for i in range(10)] + + +@pytest.mark.parametrize('include_traditional', (True, False)) +@pytest.mark.parametrize('metric', (accuracy, log_loss)) +@pytest.mark.parametrize('origins', ([T] * 5 + [NT] * 5, [T, NT] * 5, [NT] * 5 + [T] * 5)) +@pytest.mark.parametrize('scores', (SCORES, SCORES[::-1])) +def test_get_incumbent_results(include_traditional, metric, origins, scores): + manager = ResultsManager() + cs = ConfigurationSpace() + cs.add_hyperparameter(CSH.UniformFloatHyperparameter('a', lower=0, upper=1)) + + configs = [0.1 * (i + 1) for i in range(len(scores))] + if metric.name == "log_loss": + # This is to detect mis-computation in reversion + metric._optimum = 0.1 + + best_cost, best_idx = np.inf, -1 + for idx, (a, origin, score) in enumerate(zip(configs, origins, scores)): + config = Configuration(cs, {'a': a}) + + # conversion defined in: + # autoPyTorch/pipeline/components/training/metrics/utils.py::calculate_loss + cost = metric._optimum - metric._sign * score + manager.run_history.add( + config=config, + cost=cost, + time=1.0, + status=StatusType.SUCCESS, + additional_info={'opt_loss': {metric.name: score}, + 'configuration_origin': origin} + ) + if cost > best_cost: + continue + + if include_traditional: + best_cost, best_idx = cost, idx + elif origin != T: + best_cost, best_idx = cost, idx + + incumbent_config, incumbent_results = manager.get_incumbent_results( + metric=metric, + include_traditional=include_traditional + ) + + assert isinstance(incumbent_config, Configuration) + assert isinstance(incumbent_results, dict) + best_score, best_a = scores[best_idx], configs[best_idx] + assert np.allclose( + [best_score, best_score, best_a], + [cost2metric(best_cost, metric), + incumbent_results['opt_loss'][metric.name], + incumbent_config['a']] + ) + + if not include_traditional: + assert incumbent_results['configuration_origin'] != T diff --git a/test/test_api/utils.py b/test/test_api/utils.py index d09f66c9a..a8c258fe9 100644 --- a/test/test_api/utils.py +++ b/test/test_api/utils.py @@ -1,6 +1,6 @@ import os -from smac.runhistory.runhistory import DataOrigin, RunHistory +from smac.runhistory.runhistory import DataOrigin, RunHistory, RunKey, RunValue, StatusType from autoPyTorch.constants import REGRESSION_TASKS from autoPyTorch.evaluation.abstract_evaluator import ( @@ -113,3 +113,24 @@ def dummy_eval_function( def dummy_do_dummy_prediction(): return + + +def make_dict_run_history_data(data): + run_history_data = dict() + for row in data: + run_key = RunKey( + config_id=row[0][0], + instance_id=row[0][1], + seed=row[0][2], + budget=row[0][3]) + + run_value = RunValue( + cost=row[1][0], + time=row[1][1], + status=getattr(StatusType, row[1][2]['__enum__'].split(".")[-1]), + starttime=row[1][3], + endtime=row[1][4], + additional_info=row[1][5], + ) + run_history_data[run_key] = run_value + return run_history_data From a676c2b91bdcc77462a879e71ae79c0b40b0c26f Mon Sep 17 00:00:00 2001 From: Ravin Kohli <13005107+ravinkohli@users.noreply.github.com> Date: Mon, 22 Nov 2021 13:53:32 +0100 Subject: [PATCH 03/10] Update for release (#335) * Create release workflow and CITATION.cff and update README, setup.py * fix bug in pypy token * fix documentation formatting * TODO for docker image * accept suggestions from shuhei * add further options for disable_file_output documentation * remove from release.yml --- .github/workflows/release.yml | 33 +++++++++++++ CITATION.cff | 19 +++++++ README.md | 60 ++++++++++++++++++++--- autoPyTorch/api/base_task.py | 16 ++++++ autoPyTorch/api/tabular_classification.py | 16 ++++++ autoPyTorch/api/tabular_regression.py | 20 +++++++- docs/extending.rst | 2 + setup.py | 2 +- 8 files changed, 157 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/release.yml create mode 100644 CITATION.cff diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..69cd8de26 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,33 @@ +name: Push to PyPi + +on: + push: + branches: + - master + +jobs: + test: + runs-on: "ubuntu-latest" + + steps: + - name: Checkout source + uses: actions/checkout@v2 + + - name: Set up Python 3.8 + uses: actions/setup-python@v1 + with: + python-version: 3.8 + + - name: Install build dependencies + run: python -m pip install build wheel + + - name: Build distributions + shell: bash -l {0} + run: python setup.py sdist bdist_wheel + + - name: Publish package to PyPI + if: github.repository == 'automl/Auto-PyTorch' && github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + uses: pypa/gh-action-pypi-publish@master + with: + user: __token__ + password: ${{ secrets.pypi_token }} \ No newline at end of file diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 000000000..c954b4c7f --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,19 @@ +preferred-citation: + type: article + authors: + - family-names: "Zimmer" + given-names: "Lucas" + affiliation: "University of Freiburg, Germany" + - family-names: "Lindauer" + given-names: "Marius" + affiliation: "University of Freiburg, Germany" + - family-names: "Hutter" + given-names: "Frank" + affiliation: "University of Freiburg, Germany" + doi: "10.1109/TPAMI.2021.3067763" + journal-title: "IEEE Transactions on Pattern Analysis and Machine Intelligence" + title: "Auto-PyTorch Tabular: Multi-Fidelity MetaLearning for Efficient and Robust AutoDL" + year: 2021 + note: "also available under https://arxiv.org/abs/2006.13799" + start: 3079 + end: 3090 diff --git a/README.md b/README.md index 5d7773e6b..f8565d085 100755 --- a/README.md +++ b/README.md @@ -1,14 +1,14 @@ # Auto-PyTorch -Copyright (C) 2019 [AutoML Group Freiburg](http://www.automl.org/) +Copyright (C) 2021 [AutoML Groups Freiburg and Hannover](http://www.automl.org/) -This an alpha version of Auto-PyTorch with improved API. -So far, Auto-PyTorch supports tabular data (classification, regression). -We plan to enable image data and time-series data. +While early AutoML frameworks focused on optimizing traditional ML pipelines and their hyperparameters, another trend in AutoML is to focus on neural architecture search. To bring the best of these two worlds together, we developed **Auto-PyTorch**, which jointly and robustly optimizes the network architecture and the training hyperparameters to enable fully automated deep learning (AutoDL). +Auto-PyTorch is mainly developed to support tabular data (classification, regression). +The newest features in Auto-PyTorch for tabular data are described in the paper ["Auto-PyTorch Tabular: Multi-Fidelity MetaLearning for Efficient and Robust AutoDL"](https://arxiv.org/abs/2006.13799) (see below for bibtex ref). -Find the documentation [here](https://automl.github.io/Auto-PyTorch/development) - +***From v0.1.0, AutoPyTorch has been updated to further improve usability, robustness and efficiency by using SMAC as the underlying optimization package as well as changing the code structure. Therefore, moving from v0.0.2 to v0.1.0 will break compatibility. +In case you would like to use the old API, you can find it at [`master_old`](https://github.com/automl/Auto-PyTorch/tree/master-old).*** ## Installation @@ -33,6 +33,50 @@ python setup.py install ``` +## Examples + +In a nutshell: + +```py +from autoPyTorch.api.tabular_classification import TabularClassificationTask + +# data and metric imports +import sklearn.model_selection +import sklearn.datasets +import sklearn.metrics +X, y = sklearn.datasets.load_digits(return_X_y=True) +X_train, X_test, y_train, y_test = \ + sklearn.model_selection.train_test_split(X, y, random_state=1) + +# initialise Auto-PyTorch api +api = TabularClassificationTask() + +# Search for an ensemble of machine learning algorithms +api.search( + X_train=X_train, + y_train=y_train, + X_test=X_test, + y_test=y_test, + optimize_metric='accuracy', + total_walltime_limit=300, + func_eval_time_limit_secs=50 +) + +# Calculate test accuracy +y_pred = api.predict(X_test) +score = api.score(y_pred, y_test) +print("Accuracy score", score) +``` + +For more examples including customising the search space, parellising the code, etc, checkout the `examples` folder + +```sh +$ cd examples/ +``` + + +Code for the [paper](https://arxiv.org/abs/2006.13799) is available under `examples/ensemble` in the [TPAMI.2021.3067763](https://github.com/automl/Auto-PyTorch/tree/TPAMI.2021.3067763`) branch. + ## Contributing If you want to contribute to Auto-PyTorch, clone the repository and checkout our current development branch @@ -63,8 +107,8 @@ Please refer to the branch `TPAMI.2021.3067763` to reproduce the paper *Auto-PyT title = {Auto-PyTorch Tabular: Multi-Fidelity MetaLearning for Efficient and Robust AutoDL}, journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence}, year = {2021}, - note = {IEEE early access; also available under https://arxiv.org/abs/2006.13799}, - pages = {1-12} + note = {also available under https://arxiv.org/abs/2006.13799}, + pages = {3079 - 3090} } ``` diff --git a/autoPyTorch/api/base_task.py b/autoPyTorch/api/base_task.py index ab71a761a..a997c505b 100644 --- a/autoPyTorch/api/base_task.py +++ b/autoPyTorch/api/base_task.py @@ -762,6 +762,7 @@ def _search( budget_type (str): Type of budget to be used when fitting the pipeline. It can be one of: + + `epochs`: The training of each pipeline will be terminated after a number of epochs have passed. This number of epochs is determined by the budget argument of this method. @@ -840,6 +841,21 @@ def _search( Numeric precision used when loading ensemble data. Can be either '16', '32' or '64'. disable_file_output (Union[bool, List]): + If True, disable model and prediction output. + Can also be used as a list to pass more fine-grained + information on what to save. Allowed elements in the list are: + + + `y_optimization`: + do not save the predictions for the optimization set, + which would later on be used to build an ensemble. Note that SMAC + optimizes a metric evaluated on the optimization set. + + `pipeline`: + do not save any individual pipeline files + + `pipelines`: + In case of cross validation, disables saving the joint model of the + pipelines fit on each fold. + + `y_test`: + do not save the predictions for the test set. load_models (bool: default=True): Whether to load the models after fitting AutoPyTorch. portfolio_selection (Optional[str]): diff --git a/autoPyTorch/api/tabular_classification.py b/autoPyTorch/api/tabular_classification.py index 1e73945c3..d83f1dc01 100644 --- a/autoPyTorch/api/tabular_classification.py +++ b/autoPyTorch/api/tabular_classification.py @@ -159,6 +159,7 @@ def search( budget_type (str): Type of budget to be used when fitting the pipeline. It can be one of: + + `epochs`: The training of each pipeline will be terminated after a number of epochs have passed. This number of epochs is determined by the budget argument of this method. @@ -237,6 +238,21 @@ def search( Numeric precision used when loading ensemble data. Can be either '16', '32' or '64'. disable_file_output (Union[bool, List]): + If True, disable model and prediction output. + Can also be used as a list to pass more fine-grained + information on what to save. Allowed elements in the list are: + + + `y_optimization`: + do not save the predictions for the optimization set, + which would later on be used to build an ensemble. Note that SMAC + optimizes a metric evaluated on the optimization set. + + `pipeline`: + do not save any individual pipeline files + + `pipelines`: + In case of cross validation, disables saving the joint model of the + pipelines fit on each fold. + + `y_test`: + do not save the predictions for the test set. load_models (bool: default=True): Whether to load the models after fitting AutoPyTorch. portfolio_selection (Optional[str]): diff --git a/autoPyTorch/api/tabular_regression.py b/autoPyTorch/api/tabular_regression.py index 6f0f22d24..a68990732 100644 --- a/autoPyTorch/api/tabular_regression.py +++ b/autoPyTorch/api/tabular_regression.py @@ -160,6 +160,7 @@ def search( budget_type (str): Type of budget to be used when fitting the pipeline. It can be one of: + + `epochs`: The training of each pipeline will be terminated after a number of epochs have passed. This number of epochs is determined by the budget argument of this method. @@ -173,7 +174,7 @@ def search( is used, min_budget will refer to epochs whereas if budget_type=='runtime' then min_budget will refer to seconds. min_budget (int): - Auto-PyTorch uses `Hyperband _` to + Auto-PyTorch uses `Hyperband `_ to trade-off resources between running many pipelines at min_budget and running the top performing pipelines on max_budget. min_budget states the minimum resource allocation a pipeline should have @@ -181,7 +182,7 @@ def search( For example, if the budget_type is epochs, and min_budget=5, then we will run every pipeline to a minimum of 5 epochs before performance comparison. max_budget (int): - Auto-PyTorch uses `Hyperband _` to + Auto-PyTorch uses `Hyperband `_ to trade-off resources between running many pipelines at min_budget and running the top performing pipelines on max_budget. max_budget states the maximum resource allocation a pipeline is going to @@ -238,6 +239,21 @@ def search( Numeric precision used when loading ensemble data. Can be either '16', '32' or '64'. disable_file_output (Union[bool, List]): + If True, disable model and prediction output. + Can also be used as a list to pass more fine-grained + information on what to save. Allowed elements in the list are: + + + `y_optimization`: + do not save the predictions for the optimization set, + which would later on be used to build an ensemble. Note that SMAC + optimizes a metric evaluated on the optimization set. + + `pipeline`: + do not save any individual pipeline files + + `pipelines`: + In case of cross validation, disables saving the joint model of the + pipelines fit on each fold. + + `y_test`: + do not save the predictions for the test set. load_models (bool: default=True): Whether to load the models after fitting AutoPyTorch. portfolio_selection (Optional[str]): diff --git a/docs/extending.rst b/docs/extending.rst index 753c494c3..8f0d11acf 100644 --- a/docs/extending.rst +++ b/docs/extending.rst @@ -5,3 +5,5 @@ ====================== Extending Auto-PyTorch ====================== + +TODO \ No newline at end of file diff --git a/setup.py b/setup.py index a38c9c811..eed11b76f 100755 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ name="autoPyTorch", version="0.1.0", author="AutoML Freiburg", - author_email="zimmerl@informatik.uni-freiburg.de", + author_email="eddiebergmanhs@gmail.com", description=("Auto-PyTorch searches neural architectures using smac"), long_description=long_description, url="https://github.com/automl/Auto-PyTorch", From 7e67e56b5156297e1184909acacff64987a602be Mon Sep 17 00:00:00 2001 From: nabenabe0928 <47781922+nabenabe0928@users.noreply.github.com> Date: Mon, 22 Nov 2021 14:29:39 +0100 Subject: [PATCH 04/10] [feat] Add templates for issue and PR with the Ravin's suggestions (#136) --- .github/ISSUE_TEMPLATE.md | 48 ++++++++++++++++++++++++++++++++ .github/PULL_REQUEST_TEMPLATE.md | 38 +++++++++++++++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE.md create mode 100644 .github/PULL_REQUEST_TEMPLATE.md diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 000000000..29f3afd6c --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,48 @@ +NOTE: ISSUES ARE NOT FOR CODE HELP - Ask for Help at https://stackoverflow.com + +Your issue may already be reported! +Also, please search on the [issue tracker](../) before creating one. + +* **I'm submitting a ...** + - [ ] bug report + - [ ] feature request + - [ ] support request => Please do not submit support request here, see note at the top of this template. + +# Issue Description +* When Issue Happens +* Steps To Reproduce + 1. + 1. + 1. + +## Expected Behavior + + + +## Current Behavior + + + +## Possible Solution + + + +## Your Code + +``` +If relevant, paste all of your challenge code here +``` + +## Error message + +``` +If relevant, paste all of your error messages here +``` + +## Your Local environment +* Operating System, version +* Python, version +* Outputs of `pip freeze` or `conda list` + +Make sure to add **all the information needed to understand the bug** so that someone can help. +If the info is missing, we'll add the 'Needs more information' label and close the issue until there is enough information. \ No newline at end of file diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 000000000..db82b6f86 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,38 @@ + + +## Types of changes + +- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) +- [ ] Bug fix (non-breaking change which fixes an issue) +- [ ] New feature (non-breaking change which adds functionality) + +Note that a Pull Request should only contain one of refactoring, new features or documentation changes. +Please separate these changes and send us individual PRs for each. +For more information on how to create a good pull request, please refer to [The anatomy of a perfect pull request](https://medium.com/@hugooodias/the-anatomy-of-a-perfect-pull-request-567382bb6067). + +## Checklist: + + +- [ ] My code follows the code style of this project. +- [ ] My change requires a change to the documentation. +- [ ] I have updated the documentation accordingly. +* [ ] Have you checked to ensure there aren't other open [Pull Requests](../../../pulls) for the same update/change? +* [ ] Have you added an explanation of what your changes do and why you'd like us to include them? +* [ ] Have you written new tests for your core changes, as applicable? +* [ ] Have you successfully ran tests with your changes locally? + + + +## Description + + +## Motivation and Context + + + +## How has this been tested? + + + From 10df65059fc8889dd3136cbd726edcff5d156edd Mon Sep 17 00:00:00 2001 From: nabenabe0928 <47781922+nabenabe0928@users.noreply.github.com> Date: Mon, 22 Nov 2021 14:30:01 +0100 Subject: [PATCH 05/10] [doc] Add the workflow of the Auto-Pytorch (#285) * [doc] Add workflow of the AutoPytorch * [doc] Address Ravin's comment --- README.md | 32 ++++++++++++++++++++++++++++++-- figs/apt_workflow.png | Bin 0 -> 122921 bytes 2 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 figs/apt_workflow.png diff --git a/README.md b/README.md index f8565d085..a1818caba 100755 --- a/README.md +++ b/README.md @@ -6,10 +6,38 @@ While early AutoML frameworks focused on optimizing traditional ML pipelines and Auto-PyTorch is mainly developed to support tabular data (classification, regression). The newest features in Auto-PyTorch for tabular data are described in the paper ["Auto-PyTorch Tabular: Multi-Fidelity MetaLearning for Efficient and Robust AutoDL"](https://arxiv.org/abs/2006.13799) (see below for bibtex ref). +Also, find the documentation [here](https://automl.github.io/Auto-PyTorch/development). ***From v0.1.0, AutoPyTorch has been updated to further improve usability, robustness and efficiency by using SMAC as the underlying optimization package as well as changing the code structure. Therefore, moving from v0.0.2 to v0.1.0 will break compatibility. In case you would like to use the old API, you can find it at [`master_old`](https://github.com/automl/Auto-PyTorch/tree/master-old).*** +## Workflow + +The rough description of the workflow of Auto-Pytorch is drawn in the following figure. + + + +In the figure, **Data** is provided by user and +**Portfolio** is a set of configurations of neural networks that work well on diverse datasets. +The current version only supports the *greedy portfolio* as described in the paper *Auto-PyTorch Tabular: Multi-Fidelity MetaLearning for Efficient and Robust AutoDL* +This portfolio is used to warm-start the optimization of SMAC. +In other words, we evaluate the portfolio on a provided data as initial configurations. +Then API starts the following procedures: +1. **Validate input data**: Process each data type, e.g. encoding categorical data, so that Auto-Pytorch can handled. +2. **Create dataset**: Create a dataset that can be handled in this API with a choice of cross validation or holdout splits. +3. **Evaluate baselines** *1: Train each algorithm in the predefined pool with a fixed hyperparameter configuration and dummy model from `sklearn.dummy` that represents the worst possible performance. +4. **Search by [SMAC](https://github.com/automl/SMAC3)**:\ + a. Determine budget and cut-off rules by [Hyperband](https://jmlr.org/papers/volume18/16-558/16-558.pdf)\ + b. Sample a pipeline hyperparameter configuration *2 by SMAC\ + c. Update the observations by obtained results\ + d. Repeat a. -- c. until the budget runs out +5. Build the best ensemble for the provided dataset from the observations and [model selection of the ensemble](https://www.cs.cornell.edu/~caruana/ctp/ct.papers/caruana.icml04.icdm06long.pdf). + +*1: Baselines are a predefined pool of machine learning algorithms, e.g. LightGBM and support vector machine, to solve either regression or classification task on the provided dataset + +*2: A pipeline hyperparameter configuration specifies the choice of components, e.g. target algorithm, the shape of neural networks, in each step and +(which specifies the choice of components in each step and their corresponding hyperparameters. + ## Installation ### Manual Installation @@ -25,8 +53,8 @@ We recommend using Anaconda for developing as follows: git submodule update --init --recursive # Create the environment -conda create -n autopytorch python=3.8 -conda activate autopytorch +conda create -n auto-pytorch python=3.8 +conda activate auto-pytorch conda install swig cat requirements.txt | xargs -n 1 -L 1 pip install python setup.py install diff --git a/figs/apt_workflow.png b/figs/apt_workflow.png new file mode 100644 index 0000000000000000000000000000000000000000..5e3f048b9cf125a80cd4c1f2bee62b8ae432f30c GIT binary patch literal 122921 zcmeFZWl&sO*EQO>26uM}Zo%E%CAb_C+ycQRxVt7uuwX%hYj6Stf#8mS>0W#7xu%Rc$LNkwQ;~g!OoR*qfu70BNojyUuwuZo01+Pei^X&Q zdEf`s8x2_rP{jzz4)6=?YjI_95U4s1<<1Na_#Mef?&TX0h%^lHgo@iZu>^r03+1K6 zwLFaX7ZH50&mTpdSgn8743SBvs)?tn{Y8yqiEJ)@f#(#UB%YxiPpjy}{?`D!I9$St zt|bE%g^&20Pwr>(+tC~7`}+GA*XR6JPiWlgC|3S|<(}^=*k<4sMC0 zt@%IRlIt@1Jiez~u&i9be2la)Brp*+G<4z$4Gq3?#AC2MRvx0R{hFWIO~4(UTYk zV;ma>1M&(g1$=mlb$e7XIo1EU46twm2Z&M_ctT$B|I8?+$%7y_JxCbiC3% zz187$Uh3vsa1OaqFv1^GFw9>sKR$%rULF}p1~0a_TO9nXOXqrxCKE>%16*XaFwIz^ zT)*ykm}`_mNQlJD%nT73xrVnjqq^WcY5I#+iN^GKY;?5L>E`#bGF`UT%O&r0L3h^a za)Z~;)J8J-gAmZk^h^*S=8ZP}b4JjEfHRq73VY)jG&uC;Nk<*8gpse7H!MlqUL2@5 zI2a`}YQj@eQeMb#3+h@~F$|`%M0uU>u1x8g)U~T5(J6PW_C}TGD?gl9`cpD8%KKSH z0LN|!oTQ@#NW#k;B=ItlJ8;Kc-a(^8Qvk| zKL6FUl`RQaYOKam(RKR=RGar@{fpsoL@Y+93}NpVB9C`(d`^1tcUX5$#$5_F1vn!>e@TRva)$>E$d?WGM9E?P*h3lq8sVzA()S6r@rDSC-_Ghd0s?Fih$b_s& z#U%h&2YjUs!fFVq58naPI!T-Y9(Mtr60@Ez!qsoJEmxdyM6kwM*lBbl=s% z_m|5CvaWK}f(2P&+~bl&CzLUF1KV=x=Dm@*@=0_ar{8%`zR~BH_Q#Soy6#}9mIJOo zDDx*0a+M?n79|9%(l*QD10$%UBWH;PRhe`v42xfWp=*A7U_(ksMVhXNoqV|1GUH`4 z+#QDQ8Nixkj9^U@#!Vm2%^S`w8-Dgv@lB}f&UmlDc<*$HyrO)k9B{pI?5>B?oX2%* zZon2$(SPC-{DEFI+jOSFXnI1ZAVT)d)`-31f(Dbd%Q(J?4<_J2eMLEbkvV=h-@LSO zqI#*=vd$>lveVd1`a~a2(D!hy3rN+pv~UzsnMldg$2d;%f z@aO(0a}s2T?Uq^zkJ@ipcPC5M#(YR|nRI$=E89Ymu<1;{zvtm|Sw|!j_M%}+xZ43@ zqrt6bq*S|%-S5t0uQ(3aiT;--Mq^_L4ePHQ_TImesjJz+vrRkib`lM979+4?PXz;T*2}B zykm#7k56kW4rE#6C(AC%zY*DAXqxbvN)+-0&k{4c$tNYeu>XmJO|Md$ZQhwc`cD46 zyY!7vG%@dpK!m+QQsw4Q8oKZ8MP2rIPm?4NBAlXsVqcDzUL~_?-D(lE$R14ZreX{`Fw9`@8lr+v-`8Z zT-1aL8t`6aN1pPB7Z;)OM$d>}^Z-X_)+WT1@ZPmFi94a3n=O2LOSV+k*b@%OIcenL z!1-jr!C-)2z*7{^169CbzQNJm`-gs;m)joO-8imPxDYNeaTKHGr;jchgYR?7WMYV3 zs-{5_%J92CKBVv^3ZE^{e!l)9bUW@sJ2FWnA z=^-*~Ow0uTC%J+wY11LzFDs3Xt&T@RBP1l`C0o#7S$#aOoNkKen z^&|ABfJ^AM(*tGgn&{8A*Aswk=lk8?T=dC4-%0b_L1u0BLb(of<;mc%===H?C%-S+ z2X5=Y7WbpUOJMI*CP0e%r(kKgf!F1eSq!F^j^${LzHgX3-lX}UQ!OV{AY@J$KSM`v;N=4nSTB%w)(s-=RGVSnbOyp5el? zh7Us{O?0Fj^tm{gM*uwSakxUiPN6*n@W)ml8U9nJ$ItFgN9T#(&wY74X!{%^O2GpHX^Ae(ps7F;+yj!nI(zj&S7TvtkENQ^8@LrtgLzTZn z;qF*|m=6kIVo#JmCSK1o>F~Rdktyv8L0LQMcuaMq9rQsX7qM$IPXNDRI$(xurRC{X zc0p35&~V768-poVfG5*0M-4m{jIJfpD#o*!h@_1&|AtFPKoR}NI}I6N;sfDOem}ObCIYp5#x=CP5l7YuM-n<$H4`?&MT;|JFdhXi!dfF@6)N$$2gG4l(cSw&U?u zl-Z=6#Hi)1TgES~()f6cv!iA5G*+WTa;V?8D*^z{e+qh&6Mn$#LbDqRfK4Q-&3pGR zHPg5`*B9NF>#{4pj!~{hKx!ON1qnoa__BR3UVa}<(V|sM?E!#}6Hv^r6VQI&yHC`U zdr!~@nJ2}^2WXe+yxcObS~MvuFxf5qrulj#Ga_F$j*5dr-5A>P_4wA-Kms+8rQzT6 z*nZ-#k$dU3H{ILpwx@A-_ZuLCS)RC9ND)-{IN*`hc5|GN8oe>dv5O2~x>$^zrM8yy zU+w?`o1P%$%e660T$%w&zv+CvefkUg_vgu}&$UbSL=p6J&<*5uU3aF-iQ8{>nG9Rr zDjVw${)Xt+cu%ZMB@7jw$W6Gh(jCU(wyU*gsh}zNqb`Mev>S!3HJ~#H!Qy1K7gBGI zx5ubGOOkQ#ZkHeX9`0|;b@F+z{Vb1S?eBz#*tpnhy@}fBiD4VS^gTPy$Xa)nd;3d7G^5{Fx z!mKj^AdpBA&`7@a#Sn9NU)Z6N3gA?>-#n+HrdBVI!~A|ZY9z1g2w?13H^OlS3Tw3Z zXdr`2Nv#{8iv7-%*+7(`w83ETSO*VuJe;%n07&cVR}lK=PMHv7Tnfa%d(R1t@qu^X zjiG$&iBH>8W%zw@Yg z2P7T>#Z>0bhGoAf0DSg(Jr8qz-nTPRRGy~a=6Om-CgNk26+E$gnj$yKRvuO>GVW#6 z>M_=wJ$~~5poZ!5wE8tj1sM4PUh+ON`x*c>eUMUb@IHuENJ!{=zOx3?&Y;+M@Tc(< z!vcP@0?pjj)z$U73z6Tk)!zFgwvKf{tFf+xM%Fz8Kmz%UZLW~^>tMN~<9v`a<+*lE zC{g|Sxi%px{N5?vbFBfQVmWHP0yS}gLP;VH%5noBFN_<8FxGZ54STCWmL)#^5oR{toIIRf&c{lS19I-e+kGr==e#we9FV6HruwI!&DZ7 z*tmp1c$&($c-Hyb8594PvTSm166D^5%IT6P1Y$FS zwkJ!lf7aVar5Sr60XRzv(xmNw8P(tFiVwuuiR97nF!j#zd6}D=JH3@QE&2XdN}tUd zOT${Y@Z5f;BFpXGUOdzlIokDkbZwUR#eV0q|D%1|eifUl@(J#Lp8(=INXl_sDtnw! zU>p(2nraM4?Z4K)B~3T+aq9ow7`6F7wx4@?^nOVG-B16ooC5&7!P5T@*Fc~V(*AEN zcY=azp8vgvlrd0`%Ox`!DPI5#*<@!>nA2GgzhM(`*jFJ&K1}qA?A~Qrw1{~x65X5u z|LGJs_tE=^py`=243$Nt_(zANaDyWBZ>SE5VBTV`4A;Z9vn@$yl#*jr~Pq7P4&vAU!j!BtH zJbQhjhDs(x5VTAfK^um>K|Go01YVOFutgEE>jm}1{@o`w0X_3e+}|~Md7KBtkt#9a z7nCFklSUNW0DmE!R?qfpE{_rn<>NvKp6ZtkTic+0g(XkV!9Skd8Y|)8sz*$}{4w#2 z*W70CG6^~>Z#38Fh?m`JTg$F(7e{$J7&rh&Ab~yIY6X;>6Sj{_Z&3F^kL@f^9Z#2X zVCyn|e8ZfN;jxTyym2->#mRE+Yf4G*J*t_9{Q=B1yrVoV=p9^~qe(9SMSGef=i}~0 z>qGzh7Ql)i^K%T*4kIXWumzl^k&<9i1Qv!WOnhVI7=I4QI;?QiC3=&fwhUVd97P8D z-{TvXC;qU}0fLnix)RBLmPpRDE`Ii27RH^%-OZ%@<^S2>?a-%^|mNVHCs(R*E3#p#D^Xd1v`55CxO@U9up4e?mic6(f-!(W7BybN(Y632yo>{*Mql1wy49iyf?ILlE?z> z{vLQHZoWj6k!1~X{&C9-C?Tkk8>K^PB$bZ0yccm^eSSBwudEH>`dm(5_p}nknx5lY zXwj}S$LYR42)p>S&4pAdvbRe3cbB*RB=U!Hpyav^-PJ}IM5onod^>cFkosKkMdLn4 z2t{I&f2xYKI(B(|b~uBum}esOU8BiNYazR+55umduo`U;R3L}YdifSNxD@T7jW9s( z@F6;Z=afO_WjcmVHI(t;0`dKgtK?Y}kNWclb!Ts~NrIYZ6(15$KvEt5e(nt*DIqnJ zscP70u*EYt0n>A%>laKY4i5Y8G-N#2!ze{<+M0;yCnti_kA8i{7gl?T#_O`qo+s5m z)mUo$Od3R$E`doz^Ux8Gvhg0hZ@*4-HO`{Q>BR-z@j4xAreM^UB6m|Mo_P`q{;}Pp zH9l{hyW1wc1uy8H<)+{_;~jDSk!hy`qxPjiNFCV;NA85~*y~&M3q5^gh5O<$l>RKO zO*gQHyc?kQZFRyXyUyrpSb>W6Qe<{cf`IL2)@-0bfegP`2?Eq~fC>tr{{|oOfOO}_ z$;4lg{NzHk~QIjD@hVn(7`+Tl6QhDnZyh*7*I__O8xR%i~^ZEcmYCPn@hC& zloAy(TnRUylq3r5N3bmzQ5Q-U5~E=$yegw!|99GFF@lNbP77E_@iLv0KNoj{ev5{h zrW%JCUvzH1;3TNj*%zIZWZO-{Hm~Kz{GbI-G6<3yhX$=Uw^iZ65G4?GC!wnaRIKd^0@7Cz*8TT@f-NkB>&^B{BU zWavf|g`;pbahLJ)teN>3*}XFNiDsoT{eA3&C(&raBdfeWDDUxKJ!r5NDx5_Nu{6M} z!wWBf=8*VZIEAN?guZEl%X@TSx^-=rD;Z*TY^RA;?I;BsSYR26J8qtE=Pus|jp6&} zP^2l~DO4-&VuBFReu+JIJN}i!_jZ+}LipHDRz3r0v4PHfc`z9&e1ef3ZfwnfcO_`B zdunWeiTqbf=4t|=$~(qcm~8t&$qk90LV*QiksE(oIrAT}#YlJ;7 z3Sc{ft6WC&7Y!z=Pb$4ja%>ecp#NOun)%aAQa8d0MaQdQ6FJL44Rz=XR7S@R7bU0s zIj2FsJDh;@Q59L?TWfwE%^V-?#3}^I!TE}5tZ11p78}Gggz2!_mru{4rwLY5ebU#K zR)yg;^rZlb!~PwCBrr++)p=yPp4h#rqh|9r9@F`RDi$BG7&J83jjDO@jW6S8wO5x& zbTne(Ehe27v40Og_lYJbFnE!`a0HJcUCkT(O{+UI*q55L2%KJOc89q^Jx(65-SH>) zKnsH~!CJj#j*6Xu&G0Mb@9uw;`{LN7jG!k8!H(+(LWL0d2bdh~RBK*HVX3IZCeue_ z#kX|pJG)Dl0Qn2jQ)O_0pjO?V$?#vh#*p+C{cs-=zQ$w0TbP@DM!SBC(wUHc*03uk zED0r7afUz<-p}fto&?iZbVIeF+!PeEsK6&vBc&2}`sptU(3r20Q1{=&kRhxW$edw} zd|tE7PM3uB=o;E_k|~hF0E3D^z4gF6Q&F}%>n*C-c+lVNljSrBc5@Dg_WeE)1;~Bi#Ev-~ zxWrUqIQ^+%<){EXQobEsLwDUCYQ9~2@rmkd{9iPnG4DTnV+X0r@kLb%q_{=Ha|GvJ zZvp))j&Uc5M);p|dL|}y5=NizCla*4i(xuoLU<7Ry=(Bu-E8?`x%xJY=jtylSuHlx zWEiOG9KFxNsYa9AJB2);>V>YZP0EW6kRy|%WUYUKTCzz@hKqlC*lowaIwRBocj)tSSsd$=CpQF25 zSkcZ`Li3w4p9|EBBs(g`@T0j`8h_2ueC@O{$RY&xIFpls#^72#H0}-IDstNF!d(F0 z$$t_055UiW!6;F6-I#AsW2t0;l`nxMK;h}>S#+hP@B!J#vzm`de_Vprc*OxJJ{1%$gAJk5ZU%#BT)L?Qd~?jo(N zXQ8*0KASCkH4a$Dj_RZ({>iT=WAjX};L! zlqHaqc~czq5Il9M5-Y)EXJlQEh(+%1HpUmiLtg1HkW}8#(|nk>a)NipxI!S2QFOq$ zFg;70UTN1Eej{x-;g^Qxjx0@VHmsr@kJUlghFo~Y6av+2vP`%9@>P2v3-xY(_yNff zjzV*uaz2Nkv*b~`G#!AkAz%Tp;8iT)rOIQVTaIdX&saBy>nvg`J{M_D`OjGDnKt*l z>Wv2^J0m_g3mMezmbD_&PWOH%!Tg{%-!GkR(%i0meo94?iQsh0w1%bc>; zvO4W-{)0nng?fI*nq9yyt9>T-Q)8pv;S+I49!&tF(BTx%g6st@B|2tqKYEAh6qBnX zc1Ak&s8#XcJ@*%vdjm=gUL)s~5V_@qOGnm{3VM+>R<^viJ<$fEKNTmIYbwoW&JaG0 zqHXOT0Q60eMDmgzl=~SNhd?+tpqI+)jh@il9eCpsZdn>fxRpsA+;{+k;n3ovzs zcgIKet%1L1ZM>etO791rOS+oHDhHn*qDSBd{E?OYxrtehy!XCn@tNf!ohM&j_jcq+ z?w8YpeWjh}IVcw13SX!RP1y3swumsUve|d1hr(@W+PHkxvY?8E`!z}NZ+))Re3Bb! zLjQ~LEL0WDsOh;=G3Ux&EN)9Es*SS{^7S#-?>sDR; zs;7HR&4>(&3XUS`_eRy>pH8U=0<5Bzr^#vU(zkc#5BgEZ&6t z_HP2xcTESa5B7>_4Cmu`CPF{j4tW)8d_r{j0K2u-lyfv#Rt z4mOzk9yW~yY_EfUvX`zOTh=2-3p6&L=-$`w#lE99XMZy`bG0$6EmW`Po#N>xN7g2m2$+{Q_!|C+9rqh#&y z!2jH_!7U`!8y)q9c>XA738y|3H&h+{l)2>?7cj`?J`90(Qu zC3uZ%jC+`(gAN|&sD`W|D=`%I?n;J{pu7QP;C}bHP@?151>*7rGjdR?$j&n{EI%KZ z!u26Aa+67KX>txuT>C*pUVYi@?{rN?`o9l>kH%Op&M(E~rz#gF_(k7vl03Nl1=7`Z zdC@+UoK+bc5A?IJ9&GZp5FhInR0q~_(E3)xaWV@C$v%Wl(@(?aPC}%A{qw*2V!(AL zq1|JF=q?VDOE7$>gopK#ytB0po?$kAh}t|Orzepq81tOgoaWU6KRY>TPQCJ{_lBJ; zc&qh3=m=aY%AQMp@1u8`PWV3SzaHA~46pevNVP5=^iB5T^=Ef9Q@{Qq%)HExJdJR8 z&Ho;z@5llWJ89M1PEb5n;AU(Vs#tAwypYX012yF+36MWNTrD>4y-IvGl+tTuA*w7B zP;JU`cI8CazI;!7VfP5nv=t|LRE!u5C;u)43NE`MKmxqO(5FFqvNPJT*3gZQn=i!N zSXb3VU5gPtp!;0NSpS9#Pp86&XCNATT8dgNO;iK-pXbk+ zRj{1F6R8X~u9-jcUP`bl)V&Y&tu7m*9KLS<)c3_%ouk19$d(l21;*KbmqSUP>7<-I zOZ9wBy)}L4K*hSQTHk8D#lPQ>UhXr_42y~RXbYkFk`)-IhtkD6x6-<%6uW*jDX)kAoUIlC9OoB`O?{ZPeO^1 z3%YxWG>#Fay}NpT7cPPs-Pz$%e>g`7?FfzhojUfXl~cDCr=REY&RfRqj_Z6% zCSVlv2`A~y;eWBklQF~7Ek2nw{qHRN&(!rPA0_?xaDNap46qDLk0}hN1}$&P@4wuR z2p$dmH|L5zIPPApMFDB`4ez@xiX8=fVLfR6nr$1-h>E;br z0@VD1I2J%qV?4#CNqLrq%uaY zPSy@`rT`K%lh0Y=zZh_ry$My(I^GxKCF}sVo(53nBo6@bX$I&rE=EhV04^cH@i!6u zgr8kVL}L|vyKm*_R6r!;QFZPLL*;*aZNB8X)xM ziOEU=gMv7mSEY=tgM4(G z2(b#t+Q4n$XKC^Ardjj(02Qf{(*dxyYh{(~iAU0aE!&^)tA*bH(ZTT`Q{^l%r9nto zn8n5ZoX6RAf{$umwD;99<*VWJcYq|U#b?vl`+(sbx zB&Ie1R32sk8OZ0ljX|GZ;F=)9>wk9==GL{?>UrAi)0gdcV+}}cG~0P01^`5a|6>4$ zaciXU9X1H@#H%ovh^AHODWQ@h!Xg?t&<5ym^tQl1kf#eu_3NJNl5km;rcsr#13PZraa%a;JO8DZi<@VQPaT^BNnWd%sAg-9%0fUpqa z#M48E8q;ZjuLF87i^GK`Kmr64iN{iU{V=c4>^l%o$>;P7B*_7Ya+rb9?mvPKuz>YQ zD4@Ri2+)=c8U^8PP;Yd{d(v3+Yro2p`vwbpU;F~tUT`7<7odwFdwcLR1dyn$0FuKR z<2;?cdu<(nNdM8|?g;2fAFJT$0GCLvd*Z|*wD&dshYS3-))*T@X$`zqj49j#V((E_ zV|bUoX7azO^1XfA)7R%x8y+W5*?WDuRsP0BE=`z>-4p~+%j@b@v?<^-dTV}?9YBSY zT=xWs`nliVxY=4<09i# zr6&Ndr(fAj^}Tnj+Cml(M#2HwulGkmIsh>_S*ktc^>Q6hfUE!-3h+1}%V1D@awi!h zfL+UrhUE~J00Elhw?x{57-RiXv3JnLhE1G#d3itDyxpTo_{Y6yiFQ6d09UG4sk(lA zSbqF6qMJx6H}45Z2uMWzNJQ_?lFr?(08*U;5Tv;7O+N#!S!X=X$IPsBdahL7)!luv z8pWm`R7~!FH%62P2qV-@pCpqS*d0@SqB+W#V**jXR=1uoZZI_qHP%APx4$GxGz!&P z-rCpN&*MQT;@N5o4$DEDy`Ikyk{FP(oj?Ssmxqhf37ADHSvAJvz`tAM*W7x0S}G>H}Pix)JB7w}6IG)WXN z4Hax?y^l^EvyytayDrby2C{)#zRa_|9ibhTKWfg(^VTJX9XBTRykIw}fe()D5vn5B zu!8P~J%G%Te+wB175*o(ALvwWv#l6nUZ=Bdbz89-KtrLFB`_s06V4qd^}h9sLq>e2xZ4$<^nj zG#&RBAL`7EYONC7q#|79X8=VldJz1x8siG>_o%g&L#flpQbZ+`aLD6UNITY^DaAMaHLGAu`=J;Pr3Q=c&^s?>$_X+kG3IRs!8_ z0o_yh*^>;LpiUbdrPsjveU#y8-?P z2&I@)8vxypB@SRSav&OkME4FzqeE%H%Qyi)PlyBj1n_yWYO66SpNH$M&)mtKrd1bk zl9G}3{mw% zWKn=`G28+-x-JU5&j--@0{)?X0l8~{ga1EoBXkGHb`V4(H3wl99#=34{&S@Nv7>72 z)gHam@mDAkwdqdmMl!Z#$)F8!=2#Lf(M^nz#kzsZA6Q|w^k@R2n_tCATF&I!g|Q<# zVv|Y_1YIxChQZ&Z2!9XoINO3h@VN~%v8@*e@wu#sXzZQ?*necLs=33eEGW$R!PrKT z0$B-imh}Dk%8Tl=zxXJ=q~;Rb_>j{tOMed}HfasU5I;{Rp__czPQHz&aNzR7<=_*Q z8Hh*|=}6s0o17UgD*6~JU!83pQfm|&20oA?%)K*&gul$@>`SYHT@-!=F{h-a#!PdB zk?1#EyWIuX9@W7<+jn&>?XNcf@^nGatu*Y!^)~7)mj!xiscgXE*G2iA+~Iz1VQUF~ zqR=9IgBStM%A_@weN1E*STuPKb)BcX_Rfc`t|#ay*CqbcBKX%$x-;u(Oc`VxQ}Cb5 zGa^v%n_7@y)EJ_3(h%tYn^{#Q3$yF93=BSV#a+P<`ZlqQz~`pQmbNmA(DDj zc;*2k5LO}^O_PSP0Q-igUs9xN#t8ZAFV{y3A>lJ!-h8Dyi25%H{08uqWI}5=ia?5R z%i9K|^G`W$-p7t2{DgG}acN}qspc<%7EAEA;ijpp3aHoAjk3T9+C9h1hHDXkk=9;L zl`vm$;>g4~<_uhF$=%_0Kd12@JFsMqm}#&aA%$om={B|^fhgq~M-~fINRhNoI*?_D z@|>^yDt!H52gu4|A~zHVHiwYR zi?ihbAG2#5L2m5Q7*Xi+mn@s7TxuqD*gQDf(a*@}kT)9z8(ki_RkDSGPIV8dRAz*g zvB(y~dv}tm8<{E_9RZ$92q))q_?splu{R9*xezYH4yHp-WTo!D(KUK<27{qL^)k*oluZg-boENMM-E9*Y+9c0S4i4uN{6>_ZUE$%?X}0R4rnN|y9~il9g0 zJHL;#D^L~v*_df=@YTAGE9yj{RC|~KG$O6#S1)VIoH~!k(VE!y~Y6{g4w2d2YFPu=w zvLDFagi7U$OJN2RB^NCqvTrg+hKG`}yB_Lf%3DLT4n6ZBpTGw|Z>{c8LYb5P&t)3p z&NwcwT(gW;#KdV9KI9x@MPtfQn0Ok)S#~^MZ~j!ra%10Se*u}eO7hRe`9*bo3??#AJuV&B0@ff`RqBK zRMoHT3FJ^y90n8o1BB78R2%{S{aZLPNTgbbLVI7i;p1o-q0Q*s*x%&|Me_EW>rnT? z7Jm{*Kf9+9s+<)kQfzxio)vqiog1Mgf%ty7t-$x%;FM=S{R%4^c`W-iz=sA4O_6X2 zR@xrj4Ht0~#-0Y3_kks=w!XpnOqJB0qVar(3Wsy|BZ?(zn%uJ%%PrpFD?ocib{8?) zqAep*#4cObWB;7iVIuD5{Ccy4Ofq>5f-0*|h)SWd%GQhveL6q?k|3~Msbqd@cna;M z{`zY+%E@VeESKD=RRh+0{-N(7RtXc+%N!r&jw(;w$fcdkeLsHyN$n#z5HDg#2MO&U zYzAOx5x9lf_u=v~4kUHpFVw5%_kieo3Zj}1O`Z&s)((V~CX42eId z_rvnC(=#n@nD~a{-h$A9B4qDzqJSRqoG`q>gZfmc>%HjSL+?@^&qnejq7wRd%1yxw zQ|xwIq-pl!3rOwfaCAIc*^$YH3rMe21}`D_fNMMxreKg@Uz8n9`@o;~qsMZ&T0eoa zfmeQUc@AxG-ZegcbyEX9lpJ)div6#jvWn%X*O=$u2-C=8hpSC;B|X3ch~N9IlTDf5 zZExU{4k@^Uz!d-ExqgFKlD_%lkVr{c zestl@8p1q#e2;JX*fnmg?F?GEEv{}Bw1rsb^Y5P<>TL02a&1`$(QAGkPCqal)!0FG zhESj+C0at5}IPMU}mb;mz^`0r~V80x`*ZG1Cc)iiFUUG$n1sA|B>3*p}Cp zbPq_lytdf$&gMm_J2!CBD5RqA7X3Wuuk0)HgEfmOMGjeZoJW`&ct6@YPpPOPN_^W) zI-bzpAY&n`HdvNj>)ItPz@%+)bW!=v1C3#cB!P&WToL`gZttT?xA>L5pHn2%Kt4?R zB>S*s7}1@p>gSeifl}I&W7tW=w@XmSO{IB{Ke?O?+4=)-W$|*og9QRObHT;;DwQeI z&HM@h7#g5LrWvUN0k~i(VWXO5i|bzolz9lq0$6%qgWU_F*VHf;z3#WzB9V*;Z;z|+wIOapdmj!qV`D(qn9mce8je%YB7AK5u z1=JD!Y>sKp8ILrao8gKe8hO4OC@M*9!M)>$*@|uQ#(5;oQ*j~BDO=3%^_W4NXHqHy}RVM!J$4(XB778IQn~wW_A!0yZdS$Yt zbFK~-EY7jo zqENtCN)j=j0aC7dy8GOBoeVZ}6^p4o44iif9fEH=-wTR6x0H2sm~i9;b8>BX%R(?> z6S`PHW4W*TFc~W>rFoBtw_g0!t(+9>oCDWNQ@Vv*;A^OXh@(x&$nN51eS(Fr1d~XJ zp9B9pO0f*|hh4Lqm(n1xJ^IYI`|f$Ok#=HMQHcz;hj8f}zHIly1=T92Ex0t(?k^x3 zOavL-x0%p1DHxH(o@t9`*54cXimDYs`6SBU|FK{xbrql8fo_*4f+a&V*>dD}+dG{z zmd)8Yg2(~NuyZqmA1&eLvVVD@acJ;?*A32`1hl2aM$*~?gr-8f?{TeiWZkk0N+@6T zeoD5YU(p+WQ>ytEc0IJRkkwod{-&?Qnh!Z7?@IH?X|D#6_N z3EToS3LGxt12OeON0cXDy^HN0am8-D*>^|zwy;EK?s0>*w4Sx(Iyu_OTgIJvxYgQ7 z$)Kg%_);uC2P_t?y7E1zwvR%L{Jkzy*FHWgz>*Hy#OI=6{s{Yj( z;mEO8{(krjXHBx~Kc+>VKh+kIcK zmvq#Z67oH3>pWiN1Ai%p`nF@Yx;kNUxFxe7L5qg<>{`Rw+a?<&wI0N4o!@OjsB_q2 z>|B+CB8cif@pyk=nLCVv?vOZ_0RH`wLi^|U_AzzS-?i*XH0EdqrGR6=RDT6<{DB^K z$6n5|AI+G9#5Iao%+N!V-7^(f?itP->ik#DK;}(hz7qy(B^WW7WRX~?l!)5O>_=Du z;aDoHGGR5lWfz5NftY4f-lh-am)xqlCk!C@g)mIaks{HeA|)O=36Y6R-wqJ)ImQ8h zLwZVflb!{-5y7aB;gnZ*i?2zc!s5wEr};)zZk&`6q{XNx8h4bKkH`vB;MEk;z zp#(RsUkD+I4L8<)ZZmhl-0)Q%&@DwKsyzfy_(Er_amK`>H#%~%2dd*UFf z7&1?NQj&G)DdU`9{a-G#whXv(I%(w+YiHN!p%!|BSWx7;QQFRT)Wqg$4S2_4m z;l@0JHhJuO`wzQ5)|A>-5qEQldEV>dqgUld?YdY40-?(hvg@}Au~6X{H>k$T9xoTZ z(XMWXC6p>&L|XbF`@UFqBjddG5EcmGM<3(H*@)7V}v60FVKN|N}Y`556Y{R1f^(-T2SHq zj7R`?7puT2csVwV?qv4ZpocL*{+$}!6pe-{k#}5TGwnA@C+-`4o<(x{QEv#q7+gOf z4ffxF@H+?=spHoHXQ^11F|#aTB-PRicq^Ix{XRr>d&RMlJ)_$ZomYlS9OUHBZ~ZvP z&6h3|FHA%J=}T&1FoZfcmwL6{1(yT*@x!XVLX#vX4@2!KOKU*hP_RSaxX${B(fdfJ zOt9O^g&n9s>GjQeCTn>uKYd=K%@0~BMP|i)V*p1OCcb`Xg`mT{Go|oAZD6COv1!b- z8C=u&Dl_L4+J*lBw)dQQ9*$|G2{pGfL;65%p}4j3iYT}(u+o9wnD2>a05Zz9tPTsC z=7Y-(&kd%9q@x{q5LQRYhv$GGQ?ZVIEx6R*qP-3Un+VV8ApiX?zX}(iGF{b)bn4R= z_h^1Z|4G@CWJK_pNS!K}yZiM_*3mvu0D*eV=qu?*_KfR0lGb>|sUn^f=``G}b$)dc ztPy63metZ@X~b?Olf{}xy}o}Sg#$FkuZLZJo6@@XE`p8I~@MI%Wl+YCU9 zEI)-IjyuHj`LH-$-H^2tBlNWiVWf4GjzxE+jAUC0kn6g~hhpoh7Ul!}c1>`5vYM;Kcw7niQ50wFdqQzfIV!wqB8#CZtj8-A zx-Z4}trGv_3cx+Y?qVMhb?=>$4C#c9Ty5lbq2$KLB8qdp+DRl-l5II~zND7-?y_yI z1~r(-6td?lE)VH{+Bz9yoafSavJ^*I*Sxgo-AJ(>N3eL!(zhG4ygDCaTlLq)?e>EY z$JEAKI9S_YwzzhiGJuXtoz)#OsUe}tE(z4#%28?0-yo|$V>)o91eL&AKxyrKnpl)c*cfs&yhO+ho&;jd(r>!xJ7LKn@|ks_ZQWi`0-B|vtJ?(c)5S|%AqA|o{Aj&k^OrZ;dN z_#nT6O3VyxFDJ#(DR$LSLMs8Gd*Dl8vIvkw`#`~PdX4nc>J0Y{f#TYMNwNj0P&zEn z4m+LJWfi*BKv_ zMA&iPE^)m58yYN}QffszD34ClKknZB4}6*jWZEd{Qw+0(D`;L`3>VEqoLHC5Dhjlz z)x+~nKhZx?9A25bmd1yYhQ?##wlH@hl_yEHkpM~@+@X2=w}dfN;espbx0!S>opq$x z$L5WDdnGReG2!-N`Ywno#Ic{;6|z`j?3!c|(*tc<{H6BiR<`B-!5|PU-omZ3RjDWe zlDPEPdXUNSg!d?woacwe*0)2Ml2JWxn0~aG|CB)N)Pq%~I|!QCoe!TEER#=H>|krk zgu#fo{)wjK!9migPhD!H0En%GqnG@;X+6yTNtl;V;jRu_2z6RpZ;=OXcNGq@0;haG z`|woS$XbYv8Gp!9S(A&|c);}G0b2?gu!opH!(t=rScj=I#jU=xRf!*-hYDxdIbtl{ z1Mh_e2t9jkmD$it9u?Nm$#uPmx{O1PufdFEBzgl?t^Y1I{Ew3ZLTkCNBkxvNi@efz zOW3ZD zqGD*BL?JX|94jVj6NYgE!Y8{Zw~QjmZ{S~Sp9z;q)u$BlO-CpYFV0<`iG3b!sPaE? zs`aE3jdlpcpe!JuYVivuU-(lv=|ey?*x?=TI{ftGq`!#t%&G&PGK&~Z#EbL=-~Q{U z|1yqy(MzobHoYc`&2q*bK_ht2M({sX0wMfJ{Gf01Y?>9Pm*S;EMkr zG#dj9MCS97iDjHPpy1pSLd_1ej8BLvI)b{|Db)LbVm0^?LIfMs+nsn6B?BKhFt0Y9 z_@oSB>F@9XUY2SQ6=2)L06E24PXspDMB@VZ=(XPMJ8LAdhx;Rc3*f76u7K~%0(u^G zuivWypZ8sC#`XrpCql>)LeIqPI%TIywaYIbfo~OJ{4IQv%IF0+eo91W0r$g98w_xq z|6Mm(fLsce!LfRksW{{tnj#`H+0z*E0r-W}zDG{zwaLlqi>@hY9$I&tE@{SK^ntwS}42$)KsDh66T zwFQ&%5NNQ%(Jh_NASxvFO+de*ezCX^gQyNqvDoaE^!Q$he^s)HIrf2K9Ku9q37rme zsR%L|lmub8ff6;2DYb!bvns!&2F#N;EKdZNHx;LeK&cNE-twzk2l&32`lYyVPC(X{ zpi(RIj$sHdc=pvp>i@&uTSisceQ%>15TqOF2I&%&l15UxB?am34#^GDAxJmUBHhwR zh&0mD-Ce$G>+}5n@qRrY&Kcu;U<@{!d#!b^m}{*$uj`s2vIY3w)8VM}MeD>#AnAAor zR?&a@AbR|qZ;g}kX{y%*$l<>Ef3s^15Hu%U=e+QeRzu`|0UzEa0 zdwahC!V+Lm#|vnI^Z2=DK)<7^!HAUjW2pSg9pc4B&W{_vzh69))>Xnb5g-g^}^= z{m)vgcO`BI3+4ddxNZZU{SBl)J~Y=ELpp#sU#Ybc=xr4chlfe^y?b4RLwZ3%BJu}~dBm~=>rEiZ@#jMf04=~|0>8jm_Qzxh*swYf zRk$FqHfYqs$o(Mf`=_ZQ-~&ukuph=(?Q;TRfZ!8>H246k{k&08V+xCKcW%3TE%NVpRq#5 z!UEs&{h53_MTHq+FqoL%y|-?ww|12eIt*vC+OYe`pN!Z3>&%7^Q1Oce5UHW(@qeaL zgd{DH@K71!#izSlh?6)&nW%!Wt*xO z+>62qSVByPQnJCSg{kntWTu)=z6*fao@gRx*_(I%%5o~@xlK6G+%vEBQ<*5$#uL83 z%r(9FnD}RD!Sg|`z#$9HM2Ae2?G-+qfxdiaLu95|w6!{!8#mdJAvkzXf5E^BaQwbn*rp z2eI)tI4w+$>1$NzM*@RS%gET5Z?g-;YRV1T%712=D`G zp|3j^L$N6PE8uaIm!6~!@w)C3t$$B{-`6LdhecDvfJ*jOXO1jt2~c`mWy4RB(+TM*TouUFfqw@2gcH_Y+)jwjkkLX#%9~W$^v?UGc7&ZC@PF5T zj*V&MdGvdf)iaBOx-fg5ln7X(Eh^vNj00S3#<`e8g6QwdUg6!nVzpEryGiRnZYxA< zNRGF0UI~#j-W4dO=k9b&Q9+=!)Rh^?eSj9NLEf(R#h$wszxw!OGE6B9%D57OMY%uT zD*LWVLm-Wq#SkF|NNCM0CIDu_Y@@P-L>|@}vcT4<9-LBg^Tu-WmpUhVd~c6>Q*0beOa0MBTn`p({J4)dhKE%Rpit+>@}58@h22Z% z1_+v&*Y>ruhk&I$(f)q>-jT*0XS1y*qZ=|AT0>(&DJR{kO0=y`=8de`Y1hwa$ zbJ2gLH(G}$1nBy$EVT!oKfk!z$eglH;?{v3Ms$I+60kOAcJZ=F>}+<3KpAcT+?e+E zY+t>KFohM91?pJWo~0ia6BjSTtHQ>Ky`ZC``)bIkHLGVqfxR`QS$59H{l_Mm)Ii<> z<=gWpcW3A7W&*HQ*uV{~vaYG3^YhKIYsBNdr0OT^kMr?3}NFRRgRVEpkB3ya{A^1xyo( zRWC5c@GyfxBa4B6Eez{onni$e!>cOAU4hw1*`=w%DqmRFNVI~3AEP}OcAA`VPRNle?6+e<@Xb50rQ+b^hrmHIVJ(Y20N@S%W?0qpR|L zt^M@gUxBZq33ry>q_rgh)sXfG0W++k$umYHN_QtcwtBaFPE5}|J~nr63*lC6UV?+R zdD3)!i^%n%oI|4>&BNExaPDsX@Yc6y@Dx|Kdxz()WSu0P6p@K`bebP)6q(9D4mk9R zl0|}erR+~)FyiW9aSm2i{pd?*ZY`&$s{f{|t5E5{y#nZ>oEAd*tkjhJe&h^`Jlr1s z`8Hx{Qxi(?cXeJTB6OIkjlrTn?)cd1{hSdcYr{qEvsO7;XK@3tAC15tu=4v02+dk} zST*?!9Is-Q>o)|joHvH+L{~TYIauYy$|+e_TFy9<+xfvq__il}}if90DuAz80TDN^d#}iTIV~Zlw zdF68#@nluX0e{@Px`8@V9DN_GDuAi>E&d7u+3Dh145fZ_i`rv8rQzQ~2kTc$Q^RLqT7Si?_e5l3*H0Eg@Ez9%@i09!`z zY>;m_r%yxn&<++(*moDag+Rkmc7%-&L>0Jm8gCYq0#VoINL%o%MAgG z0I=Kstk8oIT1e#w?0|l8-wq#-hL4yCF=}rUh3FBm9izdTV#WqC?VYsKq>u2n<%LT; zVi>qG(;{Zhd*!0>w&$18&25?|VKe^HaW35I}2q zKr4H3L$Q#a4v_E&*f2#>m>F=7wuOUgiAl3{kqzYqUK0D~u@0Ov5&Opg+i|P*4%`1h zB6H~I`2#JHg>#m<=ca#wtdf0yfUE(cf9m!`f_tQ^GmYz^CO{>=u)W!Ih&AU-T#Xkh zCp;{~ch&S%$l zDff*vvzRG*UX+fi14=Q}m5{qoIBC=`Dw%OWcc2%TC<0_SXp!a99F~u}zn_2b#1pp2 zm?DneyOZl+TvSB1h=|$?H)Diw_0G2^o)kJ9uM(%VITAV$+aTjoE?ozY@wz;I#6!tK z_@NR8aK-4#;0seVGj>?XE^Kq%B!2Ls<9g~?fJ3hD{U;B?xv0t5WMjgWwGO0SA4$AP*=!4GmwBX#r z!&mHZ*&G}vI~JT_Smi4nTdKTxUU=Wb;g%`?zo)x&Lrj zyGZfl2mqL1*>x@d0|2IQ`!yezFqXK*W!E)~YOES@rqDMk6%8$M&@`ogEkcoyB^-I6n(q@eWuud#Rzi2-3oDr%V zWHF4|SOyi8{{hZ*c%qTt%UfkaqVIR4jgRtdn$($^k+T2!W)xPNU|{cu>W5TENxtnH zwEQ`u-2uoS+6*VxNCK3zy!EOnU!dtI2!b{A5{$Ap(%3e?zL(X^M#%+X&s%p~$ZrBuiUCl1})G0E% zv7C@{ebSE^vve~~i3iltan5qN-cGvlX16}ITNd?xo<-<+z8l7lp=VG>6I+oJ>aH`K z;lp{KI&d7=6}p!hdf$@QHd{=%aCsZ!T|)XHl=tqA3_Z-&A4-$W$>_EpfW!A*GV4uf zc5>5M#I(AM=v+neyyDfI;E3IM#U$}HW#&B+rL&d4whqiIGv zJIs*>+oTU<+#ocbU&?8ZizP80NDcKqE7U6@-k6LKqwE%vnK4tiJP%I|+>s!=wWji9 zI#?_bUjH52Q6JZ9_Ss}+$Nfm#L|8>csr8Hbk2`U1uXQd~3rrE#dN=21J#IG8!p?-7 zL7;Cpi``OS+xyZX>7)Bw>IKJ{MFzW6ovr;v>52jW)v0_r@q{P+(eVRXbhs7g()yu7 zD{bv-)nv$KMMC+PrJ%rNhpnVxt{h_oHCE8$Lk9lT^^pE4o9OncTB>gL=J;OTn@%!x zTZ4ydXpBh=6nI2V*;*1l5?CLD4-{S~A&Po7hw_v2XxlE`B)5hS+bEt=G@usdceZq~ zTslZ{JY|h16dkQal=`K*!QoN>WE&L;u;HW=FDh;B<&bN9vV(QaYc4;?rC2pIV6!>S zw7MhVZ{0m>-OoiGafs$N(UvGMR4;~#Ua}?SLbST3WQ!4gA_Z^=*thT8W8n!TQpZB56 z^|{h5YIPemW94l|k+f-Db!S_Co~5dT5A#T3_MY(i)^aY zmR*JEZ7&quX3Ivtp;6LIHh4{t7ZmTxW1Nr zn;6xaih(JIba(ep2~Zze8tgo+*6QI>;*+FQgWHuu7qf{8#NfHj&l*jBcy5k-p6cIM z<}TC{f%qM|wg7pVAf^b#N?+j!BVkI%y#cb=$R~^GrbY%-z82qavjz0Vcd5MAn+a2a ztr3^^6Uk8H8IJN=)`>!VgRtrcR9N?28}rPxnW{=z>dOw2Oh>#cy$R9WtZo;-R?gSd zeO~P026T?R9XHi=bY|v0r`YzGdn**lVtL*m0lN#K`9ka!*3!bO7Ij4mdkJkKDwwkGib~0rOOU{L22escr+#+9_d4jTALru_cKbGy&BR{L?G--etwYo~JmWJ!%fSKmG+ngPnAxWBEB_4i5AG#A}bEbEChUee><5 zi<5PGqDe*8=gBAX!8p`xRUWizY_VfL$GeV7w(av}cghQEafa{#XfLyDt3H?$6NG)a z3e+XxZYPS`C$%dkAp>X&IKP{REpvVQ9{A;YGwb(qQXGoGFLk4++!c1D#3IDN}2 z+-*(f2*ac)C2zly$WCgDYMW)D!Oh$7v!zCtvgP788A$l0ql2D6C}D)CaoyV-Y z$ROe8`p9ho+C|~|KJHJX6Yy*~8B*N=ugPB^gxBi8CshPULX0>nt2CCjHgG12>2tQ# zB;d*h=EE++(wuemwI)GG>F?>i3_|zGNOSOYLx1D`_Up=1C+`>QF3tGa_w)w-m1-Q` zPC6~z=PF=E2bfggPtN(JpuYEc>hSw#v$A?Z-DSscW2IH=Ga>5)s?2yun9Pt;_MlCH~jP_`k#)2ef+8EVf>lBrhFaByr=xeWr? zORaq)tKY}vvvlWhXfMZKR(&uMW`x1zALuP_@HPuPZ**m@JSjsZsfZmnD2YG<{4lvQ zmTddaI~!uiKV-vqug;L%^;8G)rMeQh&O+uL0{(PqmOO zLKG{Qg>C^K>?w1XZ&S9rfpz5j3r2vLSnx0JRu>?oMy5|y>`>H9+;H!qb=|o!FU>Gy z3jKpWr+9V!YJbmDd3ye7QbipSi>DbCTR29W&MGKo?yIS-BN44^@ zY0+Wd*HRdbZuu~Xq}WhIq`dg9a?+#I;jOZp#HSD_YC%8ITJ^6C>$-(ui=HiF3^j45 z?weqtK%o+a&dRP{-lH`u##~HzfCr7FOkX6t;|R2B-^0*T)~yi-aH5>6yv(Irb!hkG z1!)tKcTe1AtIU~PFrcRNX`}eG^8b43*O%KSB;I6y)`tz3%|`g`qL2Td%QoyhoY(T; z0f9{b*7<_^-68Ff0zeA6F)|}D?b9Ywb68@`JlL>Z#C#ROjFej}OqB?vA^m-5GadM_ zc~&bTvlqKxZ;Z-*2qN_=%rTLfqWr3hfZcM#L&KpfO1E>$B#0)0i=`-np-x1-S{#0F zHOzSJ|7cG53-UazTCTwZJm%<`gQwA9SEuMtKWCoXH5o8wY*nEa5~Lnob|twFnW)9# zD7GBsE}ziKc6x0lt~`!}NN5LO2f*jEJ({84 z&fUaCooDlSOHKe9o#pLK07@Sd1?PM?5C#d!P zM6UYI_d1M0xH&3r`)1W9oX_kjiRpmgu6u5P)o3X{WAyJen~4)t$i$X|%|Rl6Tat z9hi*|sn)A5>)-7t)Q$d_pNaR=-e0=$HPJPl8~$nj2;Ts44i^kNebTspQ52ibWFG2H1P9s!j4Zo0vo4G&gr_#e*9x01u&RO=V6 zf7jkuf0bfNM4%77#l2UO{o>YxlSWrkr5Na<-l)-3=!!inIc4guW14`_QM5IeZF7V@ zJdNd&N)y9K4a%jYQXyz2>b&V330CZ=40}e_D=~>1tydh;<2$85Z(FhdDTPq-MQ!m@ zRE3jE&sz%5jrwmk5SnxGPJYj~!O2!e)8Ft9Vet4HxHroO zv^Z}R7v$a9*ORATxMTZ6ANq{#6iKPCioQqGoZsbrxom9u)IQ?5jPEN>5^rz)R&g3% zat+ZMr)OE_(ROp+X>3|AQBsQIbqfQJCCG^MetLWqW6W9CbFIo4efQQ_y@RdQl-94V z&2&^jZxJGKOvIqk8rJx-Mj=)@{{h0W#&(3q!+il8)#Z01pLS16!u@}I^04P&3%Xh+ zd#V9cesO4zntVD{7j0WI$H)X`*IcRVsZBGcY`-5=Yz5^sI8ke|>#Rw(ZxSdZ9ghhB zMcDKx&uy2%F?h5D+A_64GqbFBmd7mA^>9KOwn@Mi7l&?{)zk?dO}+V)bxpv=xU6?I zSqft4e{XNs5s+9+%S^F>Z(dK=g-^sFahEJ1>PnH<&+uP@w>`hMKZ-A4&lQ`1lU{Dc z32!a)&RlM({5e1IbdZp91MzH*_&Xf*-iJ9>6CPVeV^cLeGiDq;wE5@J46-8vUeikk=%&=Bcn2dR}NB7bR0f&kPq6%^B+H_E910Xs!kYqgP{}9nv<s+c|ZL>_fZg)RSr#(oRJh4biZ970TEFuFQ2sh0N0Foi zYTtxuQ4VEc0STK8+XfKn6-w;0f|LF-IA3^51!<@vDpRB7M}HH$$@O~Ws`MFZh5A6z z^+%_Tp&}G(c0xmIEVjYg0dvr{41=kecl|{gIAm z(|ryl7tK{qP&(14jJ^>y(s;#!Y6j`AfUe~#B&_72lu?gGOQV)8%in+XM57f&M3-*9 z^Xe|Oa--38m0t)b_oK1Eqf~mmTn93$1i@5Fwsdn5l&MYOI-S=AB(0wgo6$=_>W46 zBg|nA*8#o-;+1-&^^d9phQ(x{0U0L}eWi*MGV)iDP>O{9%EbJh(A>=e5wb2F6pO40ia3PRS;8_M|gD!;Da zPMO*>l%#2e+@o$*av4g&7yw0m=oO+kxQh zQAWfAs!?<8B$VE69~D3&>S{@opFxPh0N5V{U4$4CyjaINJAvyYX^RKYX+mPXZzgTJ z{E0YJZr#3LZg8j;HdVuJiT!zxq+n<7yhr}GT&TmJqMo5&4P=E(ewo*R(MDdtcX=xa zK(y0AXPUu5c|d2Tz)8z@-2%^tCisa~0eAZ_G6lXUOth4Cqmm znNRB@)Hm9J`;(v${|7p%xVS#-K7c-rp}9|Q+o1b$7*Kje&5RU_Mfm>CY+P$t}X#hF%C zfIOF;4i+_0&wUp0i)hKYL96EB^pBT5A4gZoOB1Ir*)6yD(dx!88}CSjaq8p*ExWd| zZn-AiR4A(nMThMOlHe7%$q`~!C}~vAT2b0vxa6dEPq{h% zti<2CD!UBN)BL-g*?G<9izki7vr z-+qH&j7_7q+iKk&M|ZPmqfR1!nE!_p=Km3g>$}6)EL~s-yOG?mr^N|f@Y5i`{UjB} zn5_6KccvJDTOHgTYmnL0<8{|L1(TNBXX9<|^duvJQC+q=5*NnTY`tvP1B#(psbKZX^|zx1_3Pjw(hU34MN?9*4?%6hpVs1LQkkQBC{t|W5qYMZ zC6p%=6a<;6U0sbQte1d7d^5%5mA+oGgGaBoHxXVSk(UqDW0r#nGv*9UAY8u;RI10F?>U?3&NsGCm7b0y6P^$Se0FOTA6J9zL|i^ z{YeUrNa^$JQQU=|Z!v#J)f%h(RD7^S;=i;n zD^Y@(=7dcB>3-pWWXH1o>`YHjjKBqw zVU7+&?k>QO6tH_L_1LPaH7j2d{a;jg2YVe0g4<-5*WF`zh1EGCRMC9wmeuO|2Z_B> zTy-W~6NCo-MWc+j<>oGNQHqi_XM9jG{qc{VvHFYZ>B0y#vQ#Z1wM)T$eq&<^aY8|h z^ACGPm!vs}`~8pddgg3`j>CS|VR9$zm@?}|$^qu8~*$nh(-_|2c+cD6xMJn38I6P*{uS!;J?xzo#7N?sP-?eZMCwWxbXrNSN&dRrU{n6dqnf>)6)x95GQ=djZEv5@c|Cgx)clZa1(LH$_j||)4 zr>>XBekR{8K&nfWC7j@28E?It&Xz@^DEFLL`*ILMQ(^eiM7=)_|9xV5Mu2jQt_!$$ z$ww=eM;cDt1vR^HzR8rUz#`Oe7PC-qQ)Ny9Ms&n>w>(|edtOkt2IE`&gn&!u=plvYN_`j{6o_B~ zTT0;%HE1R;??j3p2--+jfIDP`yjvsDa2ua)C+axdWfAAH8L_mU{)~zyL=vQd#zyVf(=*u|HHGGrnqsu5I$WCiBsi6Jfimq7)TP4t#$CviPSFmkm2M zao?vOePp@f+5^0FuCuGJ1qmY}dbzIE87(3Wt4aJb+zzo{P!@%jQ= z^oVtnu8!nuwCoj1L*+y~e)O#i^%Qbuks?iI$HhWCNciRrkZrkA;p^fuONkh*WrY`h z-yEUt1*rPzey>{#3*n&kAEq>NjUCRs7@-0aW=H)$EVQaU5uTU6_K!xY5tE>px|q?* z!J8}|7B%_Laf1T53Fl21p&DrHjMu^61{oM~Yb}Lo_jVkG=5aes`)@8hR(`Bp8O>-C zIG~7h-E9oHZ0>&fjgKXJB9V0#v!&H+iZIER`JPLUR#}4f_VdCNJp|_>tk*2-jegk; z8fW-j@e_jn?6TdumjU&tvXE>|ByoEvQNCK?vsg*_}b{5RpIE#pPd1m4}s_Uwl5y?!0kg;q$h8VTFUvM2B&y|}hk+518F-`2VggeEA~GmNZ`78bMdQ7i`P@FN_4Z!+0z6Y z2TO&T=6D>9i(w<6O3HWkD^H{=?m6H|CRglz4I=E+3wdps1Tk*+Bx<<}*?CvA4+o_~ zNow`Ol9wxJQBD8}LRxDUT#$1juSi^9x#Rqz*hfb+79A&T(KD_+eQI52JN$9b(dpp= zaL2ttrfGExCZ#nU){DGQp!3BtedbVCdWu99$4h1~he9y=IsTEOhC~uQgKnsh?0=ni zq@rOi)JJ7Tu1l7vy$V_x;yCFFI& zz>C~`0l}12uKW8hgvF~)JTzjgm5L-IP6YS5(=#T%pHsN}1_qRW)cC~*tyHDd52Bcs zoFY=imHsHi_mT&!EnM#My<~sF&I!DUMy7UXz?r0R(~SbXYqQSm-cC7tyB1NT@%d7@ z!a;WVfA9GRDARmOK0>LNYo&#)d1^E8*kS*TRo0`ynQfTpN>p>qNKtPQpRoMhFZ9SJ zL((?Tik9U8f!ONT3j14SnkWTeUMrt0cHDP&j;w#YE#D~Urw_sxe>)X@C|=qWNNo3j z>Kp6D@178umXSQMo+0N~0$Z=S=d9r?xI98_% zrf#>n6$stOt4Uv0iUp+MRksU-BL5hp*v;z^Ec@BuIjYQqf)H+X|Ow7biOLlUH;1N69a&s3)xj{LzdyXatwbDZ8T* zPq_91D~b55i6o=0Al%&-W>en;D0WXrrKUDs)ie3_OuqT`>WKS%-yWSn*!Vy`Eoyjb zx2uwL>jh+JI#fpWHnQz-?D}Q$*)OdK+{LyxKnHy;@8`1FuGy^*clyppjDOW8m4r$^2{R2(F zvPoi>{JQ#tiE|mwd!XBGb#{KNO>r%954{wC3-Wjyjp^4FZy?4};ddpm#)s{LFQ-56 z(2XtxD}Ehn^X+I1=05s0?Z)N(>9Vnz2WL#!ATYhxHgBc=Wn9RG(#1-_*RTH@xwwkw zz))%PGwJeJ?j@Ucu%NeGl zHoWHJd)i_rlsH?;jvR_aU5?1tc@JA1K8nOcz2E(Buwx$&#Zr?#z3zH*{5q6#ojD5H zhh&aB@c_e8T_RSg-31e=JqViZbts`vf&)wJ`#%AhF%`+iNxvrIiuB}_N*=5hKSvj~ zG&l}~di(!h^(1AG8AQR=pcQ%|;(MX+Dc`TjT>hM2Qw{55_`e{X-rtWQ46UhwIC{BC zEw4m_`1NYl{o50;ekGDFCBjh}UR7v**;D6%KEVSQw*Lw8^yBWG?=7=R!Y1`X(X?BT zN=rrS#D)o-{=W|R+4cbB9$UU_N8r8O;G3)I^OK83jOMtHvU}tkq-=LC5n9W3q@sqe zZ}}6m;>;5-w~+NK&ke^f2j&6UW)IX*kfs=}2$Q1Yr@&SdW8>9#Z8F@c#l|#*hYri0 zhLq@9`x%{?$LAFME_f)%i}Cc!lUw4)Ru0I8cJ$x;6bpk$c|91f8Zhr&wC{n)%L2d6 z?>RDlA2i}Qv16cs2g{&(ysN8@_n1HnC`tOS*yK)_w#?j zhcVMXyC+s2cHggO@@|a(tXUG>RrdJ(VQ$KUZgIJiu3Jbcx>fV5)4pn6-EYbml})#s zrpyG8YU4=tN(kZ+G~dbGPDiAawTG)Ke?)51Gj#XllXX;6>_+9iOL=42hLG zw!LWx_fT#>wjzlz&D0xKqO}kbMmUrIHcW7Os&rLx|F(U?LmBzQ;=z(><}dyWcp>c2 zXCn<%S_W=-Btzi@ItUL3COn|13ciYt4|qDww%k=gB=W0MJ3M-K&d~(HVq4KaKq${@ zFCylTT|Teg)oLw||BUQCzo$2hF|aGDII+s{jt^z!5$b(k>ytHo;dyjIdd!x`hG`|R z`r!tj+PvvD`_@giXmRhrSH%@(57u@dboW+N#aOfNV?`9< z&```JAMS466;sZFH46aPZYsNOHx2&|=2PHvMpNCE!H9uF)YlHN+HaL!m(yBgfX_Y_ zty2>#e`0=p9jbNR376J#1UZs1rF+pBcH+0+)o)?9-0fH!BF{>V{A-VJC1L=R9>vcO z8ztVRq34PF%|^A$D&Bc>;IZV@5nsUJ;HNv1hePk^?E(5CEsfQeqI8Fy$}O~JnOz0o z{QTZ>LS*>EzQsCDGp$O7)|4NV(rsNzeZpfFL1qs0)!Bc$S(OSS5&-;K%q}aApL|j1 zoKf31r6mGZr>)&5kRyN%-_M&oI5!h!5`fv+V?BtGH;5*o}$N-$G zT|9_&6_48(G|LxU(|`ANLLSt=kDP!1o?IEO0NQ@}Gt`IXZ;V#@+$8D_o~;vol5&Mc z^0r{4_bp@@Kx5}*S{C=({+(ncKyqn=^b|cwfgEJK{OLU*S}i{JuG0NGg%T0=q&e>W z;pxB!chr+q!s-LHpl0RE)#mN6!OF`#3xykV9OEsXqxeFft02MfF@vs)$!m@Az=>6us_V>G7&X z$W;pYV?`W-tNCn{L%W6ax2XXOJ6VE zH;kuG9vQrv9dR8kviOua_9!b^$z(UPxWrY<-bECx7a4Cw)%`?jxV0a}D}-0U4~O(! z2={RQUe@67po5#kSqE_%zZ0CMt)NN^`~DV)lw>0>O5R9 zc%`ts-%pwkkiayiIMTXu-m!~#-uz^$qWaYyxjMBm6X@J3L<|;d(Z7q820&Z% z-z&UCKTfhjKCj;rU?5M0Zdk`>pgw-Vb0j^}w)sIvYmz-vP=agXV>oy5LDAZzV{_c% zqd20YaZ&+6-b4AX8Pri@20^+*hg&8s3L);$dG4EB&EbqqCXEASEBnd7T6){x-sroD zx5@eyei3?DtEBjYZqbJGc~c`fp1jA*3<(72$V^I)Vr`HNkFRs2glPGqyu@=`%Z7Xp>eBedI_$TETq^?NFJDI+3%( z<|q%nE+EOpsPr7y2I*DLb*`HpRvKQk;V+$*uCu*&=QXGU>DX`uU*vzxHC^ILrV5(6ZUJxy_3svvyHwIn2?{cDTaA2?V#S+p(qyZ}~I_*9Wuo5g3JUEFBaE}c=g94W7H=CrwWX&afJkLTkLW^c$@ zujKAZAI)i#N5Y4!-e~w9_zRJLR{Q4^{DpeJrgTq@u0at7`>)^G^#qxCS9VjBH*Mw# z-}`jd5iUa?4zym3Jw2^DV|eHtu(;+cyG(g9VsP)(_OtD^-xbf|Eje09wdk1<-`?GW z_82^R%hm`tq2Cxhg1a%v2V_0PD1tv$KXxEjo+Wh!wr9=#`ZRG*K6ri3+hv>W6?Vf_ z(KeohHw82j$r3ZByb+p&_Csd6#k&#YjtpDOGgogc5ve{ry!7%x_k5`?k=)|P`Zc)_ zpJo=bqj_Faa2suNWZ3nT`^NBeW|dNt4J5ky_GwC~?Ug>!o{uRUbui0H{yV42o!8qO z)xLMOzfa@8faEj2%m)v_O8#yWkT%Jok4+%<9(% z#JUSs5g1~YW3nrnN8IO8gtO4>%oJH+Kam_p*Y|PUm>@x-W9OLL*C>6n*K*_xoocO; z%e6pN*RgnG@W>o$Cc};*^eeMSokk_RWDsv%=`WR`MJ-qY>S?-fu*%p^kVd z#RgUukOJ1iS=W`_m>RnF$T#H&ez&TZd@y!GhH}bNvy%;lM2J(A+AYtLkaT#{WqA{uPnjS#NFc@nQ?~ zFN`GaF4=e^C+r2h_Ke@}{d44zCR&kYS9A%cS#uL6=+9KD#pUmMrQw+IQ&PL{Yjn-q zq($duFekt1=9>T!`4ASr!H$7^)y)lLQN~esc;AtQ%7MZ9R%5ap*@97Ff|~;Rl!%@W za<-m44p!LdELUeA0-_EmGrBePGF=A|8Gq}wR;g01ts(juIwXakx0Y!DE{p``&j=P< z0Zq@ds>HijTf7#O<-c(QO_|Oa5R+xyy1qE=Ybb2^)^%asRM#&nEVUk`{n2vd=`{JI$M{e5p*Z05=Xcc^Ailax-BaE3VORAg% zBHc(fKN>WJn->ZN_^XSfpa-T*wk&ZXUXmhrTR=-$z(1U?;5*2Ms5MbpBH(HX@c6BmYkl!dy ziKi;G{HQRnbM~M|eb$lKMv;7Y>Q8AuWwDl+j)rGh?T$lVHp@blXRa>s(oKr;$cb>i zlq7t1<1+R`B4JJM9nw^lgAbX%maXyDz_(5TC!+A|jk*P4;pBfN*?h8ttEA6WFDh-v z!~IupG9GYb!sE4}fy~A0C1_&7*=;o4$gBIz5_0jivKNHMx${q?7A1)NlP#;izNe3z zyvxH@!B@}C8EH|ZbdI(SvFsMdzmtLUu!`=qOKDO@v%Tj;R$pBbV{TW&#)+KCk*jFk zoG-%I)+8E&3oyqJM34%N<~{Go0Ekcp03^~&%&7u z%n&+Wvju5PDK7ff4gP0F0@bFqes783$88N_t(aOX`!~347!`zdKH37slO#e?C-XiGLTN( zUc5fF3W_Q}|5) zo;tIgLHTM=z?0g}&tUj3ZzV3`K4#GdHMG z`M91;dE%8Qe$woZ5O~-s65U@LcumInVG}x`nNGOwY{-$6E#eb#d5up|tY4LTGMg3T zqmW_Y>^bUl6E@Dri>AvqJ1@kNptd2FI^-_gC2fcTw6+kRz~sED7&(+KEW56 z|FyW>%&e2Jkm?{GJ{{_e+CIEIzc>!}_8K1hMMD_cM%TsRDQ3hc(^LdHhSxrhy50sL z+w1$_yD}FVSN+1NcwsbPoP$NfuFcA5ThV-tGVxK@{*bz|2$)oJ_+;*p*m!pJF`)=ag z&<(fVq}DUInYP{nAocp@4&f*62RjPIH*(STmh%Gc3y=Dt7c7$wi96AX;SdSVU>ddU zOymZo@+-qZ{TteQURP8zW4oFC(h(^X@v?Vv4q6kxWa=*kC3#p^H|vlRds77_4hl2w ztT%j39m6W*?{F@!$Hg~<&T?~;7Yl+`oNt#JlM94~4Gw(k(hqKAe!>CoPhuSqrr8mo z9&|lS8~qd|@csu)*E#*u?8(qx?&o>J>PMGrdtc5ayNDJjXXkw4@y?kQ{J63tGTDsA zvfG-h_-9CEsPm{z8#9_{y@G~X^5sgGNz*s-rD58~t7p$Xw0|XMicpn<#ge~} zGp=`(l<3y60@_GtSp}`GkTl^d#oR@=*M)r`ukx9XpPEuyK=uSS0|il}?6EYN@yB=4 z9%DixY#v&fF+1w@wkZtpSp|>$yNnl`5_j7)h&r1L(G^$tCrX?t6#5 z03poPqDY6f2ANsbq@$N7&BvH}y1z9w`gro>JEHBTLkE!5$)XnLT)J56DqEeDubj*} zoVa@N7nNN$I-rG~*3Lp_((R6J0t=rENuOJaOtTC)jBT}Up^I3LK0zsF{Z+_wl)z9C zsPOMp8O%M~#D9EY@T%reG+aD69RZuVnwNPZT;lCpl442i)jD#urUz6;23tRf)G8xi zV-`by+5R8)-ZCr;=Gz+u1nCAry1S*N8|hLyMH-}}yBq0}?i2)RkZuvAQ@XqR9n@!^ zXYc=eoiFD)-}Vx9(BY^UcvYQ!gBwc!dDOC0u0N}ixWy5yp>TQ_>@WxD9PGap6xr)Up4%7C0sy}U1!?`-2iR1_vi0=jdF-X8-fx~uJp$zk{Znn9b@Zkpvf#> z!WnU2H%bd(oTE6V-5a_i*yQpWNb&Md`R|Kc!S;|k8$_N6XUyju{P_6+jj{YDAStbrUYbH+0xYS_A3r!M*l|$W0sFjm!QoZ=1 z@$njmS;o%}-iPpI^-&%Oc#%Bt;5H1!_rW^u#>%F9#J>Er zx5}rRvp&uf*Vl=XL+FBZf#bYRO#W#RECalLB`=qE8b0UUsZx;+52zR8b*=6If7`Le zBU$7BuouFao@Y^PG7SJizQlnxDTX83>-mpDN&g zSZL-rcOkUHeU&-p27kv{k~kM`TD~t|05z&@yz^?A0AXQ)qVhsh4B6*I{Gaxx5*w4gB4x74(5) z901#xedgKPEoJwyUnMBQQok}frh^>&ij-kszoBv_bZ_V`spoafWP(!8%Orz zf#G;f(X^CD6zTc%WCSz#Nx1=Mh?vjzndD}}Kk>rm@7S{SA8~T5c8rTdE4_Hnr{sYx zbBr6Kj7J(eJJlu7PWM;=U15g#prg>!^gNJ-9rps!IyY_ERB`_o3^Z4|-&nc6RecS# zl_g~S)X{dU&Xw@F2ZZk><0hkC8J$ZDS%t?)**^<7Kwfze)6>FC?I_4W5(iw*4$m`XD9I43(~#@gC*b+T&ok?Z_0hi(r|(7nox~=7B@i) zXArNM`QWcmX)sJXcW(NyK38AJa$7qlm^jL;Iy^lSq%Y>-s?#F890W{ih=mdcugL(k z>T6|NvTB@vTw~0&IXFYT%(kAQQ9V z4;XhAu|dEW|LUVfmA<53=ee;!Cpas*y@0;GeQnmW#k}{%T-I}+?mozq=H1SUDqh5s z{*5=_?-dPi0{Pklio|1%kaex09i?_*aJ%bvl!>1Wxsx?YSbU2p!VZus1oBAYq ztI%Xt7-)%ui|-NH$EP^LmijeR@Z)OjHn_SY6LqU1C(Px%t?y7dvs2Sr5b{%s*e~Jh zzIq|&BX5N!oE2xvew<_J(_`No<$?&&TI}--^~-V=y#3Y3`mfCmgI-8}zRxUOy;C(1|_uKv^kc+K6 zTe}I0G-jPW24)NM>p~-PaxIpFa$LWZHh6~pmb#lM%k~P!Q^>3e)~OB-VkgDD(Xg^v zi^X$9SaXw%Nr>ndE;VZfALnV%AKYe*W+xvp4(%h)E z@2r-ypHX%uLTU_PBVQ_b><$uyDWt5&;jgX2gRpEcSU$)$$NXP$lPbw#1g;MnH}>n+ z^*^@a!eI5B#0_>JE4-txe`clMPIuc`Fy~83l0>z6h%(^Xku~YUHsKl|7J*XieJ1O* zB(3bZmqkchv0Y`Nl2=07jig_HO)P8$NA|ANF_g%_v#cUXHlr}}t~(#Sg1`u>csW}> zhv^@h9_||5yRlAHQ34;ap&errtt)Q~F^Zy>$2MCo04@^*c~x{JZEn+KH~D`qF3m`8 z;AgekFdqL(_aqvvT5GATf_~ZPwd8DzTVHH3?( zC-r-BuSuDi?O)cvV$JW}Ses^?0bZ{E~; zex-vjyHKOU(R@$r?Od=}_G{&zVE8=5+gcTf(GM=K^tN8A&vbwIM$Va1BlyZ`>it2< zn_c72;U)%$A)r+7(gyCI``btP$jt$(vR*@hL!#*vz>bK!%F{sJ?sTQsy;+~h7fXg}p;flLRS~RZ*Ga!TI+J}s$zBy}fAJwspU%(Pf|{WyYq7diyV6)VkeZ!4>&$BNC%TN3=F0 z{;VZ^LOWJ7-1G6+SYPEBEwusr!>&4AE3BK_LEu`xA*=yS-A_(DO{M?=%b>5ovZosYHg_yHRseOXB zp`wp)e))Rk2#=^-qS}heK~(AVZo!r%`8gDtNl2Z&E7!!32P%I`oTP3SGhz3r&9dap zQ+i2n3a4)6yRVN`(HmBz;xxLt`DvF^sNjWhc-bm`3HR?$i8B&y4*y=vlll-7{3OTf z4IkDG{18sA`x-KU;rymkx@DTWK@KB_mlmd(YBQJW-VOvU;cWHg~br)RpFC0N9 zH#0Zb7xEsbP0f1P*T-yE2dJN$%8aTVE5SwfH>7^xIpW~EZFzjYV7r(|o^@Kx8}z+5 zGw&Ej8Z*lDAdVlV!4(;*a&ev1j|{1iTV1DyBo4CFSU=rZC{Iy*gfc@VNzTX;3{GRpj)ERO9|GZ`xYOCuCREBz9F+hee5la6v)MY2of^sKM&?Y1ZLrlR@av;xsnBj+P(&C~>t z5kh+4u8dGDwMi6blI`@ZpGB!W4Cy=~6(15+V&l}?7i-H<-?a6TC#*-z@|M*r^Q{)y zOg-CM-48W1YTV;JWA(j8ldZ7kUB_G9{ycINF;%Q(xQr0JDI=JfW>$k)k&QBu6}9H4 z?ws2AmJY$tqt`)|%Ss^@k&u%Y!+f-SNP_X_O{^(~YDvg29dyT190S|Oo~JpV&9b!) zm|16+y{UpAMkFQ4>F5xYMy|PY#j%JiH$&TXk8Q?Epe0go1~n1TinzitJA4*w+9&fD zi=91?YKkGLhA9f)8PMm-AIBHc2*Dj`(+CMrck<4bG8^qcjpr{#=Jc1#z`&t>%~P@D z&lDD1~?2+a(X=oI7KFwNJ@sEW(=iV_Uhl8!>IWYjf@!HiIwJv z?O;EUv_OfKsz`NcESHuyYeY=sQ+hb&vEKXMGGjESe{zPRsN9+7tgA~u{-!=wjbHYQQEQ3g)OYQgp>`PtKg8=qzzyeNlIM^L(izfD*8goXZHp(2wfl+SL1^S zLy%i1ES61$TNfkCd1TvZY+>NqFS4UD%;4gx4I>PpnD7^joQC<+F~d6iAZ`bsji4pdlo;#Fk@F&FcBk0@!3Tf522vltp^*LckYz0zqixy+4q@YP9_>8H*2T0%~B{vxTTkQkMi^HqJV zFid_HmGC0xe+R77j$B=8l_(>Q3)1EVpPb;+~CW?l-}{ zzIY?U024f>vRY7Qv%8rfbI%aLJfkr;)xqg$B_JV^pP;_woUi_o=JsWIwYKC>60O=# z6OPr+h-i=t;{`pWMc%ckyuC3r*wt6pE16qg6F1LxJ26Q@bPm3@b)Dr7Ln1`KeKiV% zh+aEy>gl^34qGNSY)@CDSE2!G`s}~PTZCySql{vyl!qqmsYO=k^yJ(x z5u4=#0;ynIh-N37uEVI@s4Z=?@inpyoqezwSK&E1l1JW+kU?K{WAVT4eKA4B5q^w0w=Y8(AI9< zs+AO=l^;MJKP%BP3FeXaMKza=GCHF?y1xGEeS1e~V7K64QB$fdRe>$>aEIc_t~AL~ zmTz`j`ZU>Z0@sbFD<%nwqD%#<^UzSqGC=taqW!YEFL#!rH`;Q&qPM%QIuoV2yU@z# zsvSc}vIT{UvjO_aE(aQZ3f#RliPGp6)&#y9uZk~NB+7mb@4*06&Jpc3W*kNvV*>>K z90^?KT&W{Ux?PGG%xYA6wo}|(e7XUdX~#DlJR`=b*{ocmK_-A)%LXw4yP8FOtD6bsaXruOaW!&}IVh!9_LF0$rZz!}cFIK7?WtbNH%PlO3j$=qAQF340JIN~GHGUsIglm18vfhUl z-O~M5TQ!#QdXoEPD`RL<0t9C_enbLD0Hev=!UU_=>1G6O=dVS;Fy3l1c00x&gR7k&v-%>I<79WSNKdl%x^qeJowo@K!-S!?P9;e zH<;89Is{N+bKPahHHOsl-aX3Q*c2ir_$_e% zz7i8j`r&mVc3#SPSEq}Y;my4fx8pU|JG?NJE99X$b(G#!20W`>fK>H zqIJ>XfN~|Gr)E9Aqyj<5f>jBtm%9`6^-Xu_epX*NnYi2wAB&H*dr$=-zst5zkSuNx4&hmf`(6I zr19b>vQiUOwT5ElfP#Ac4UIg(Y?xKahLQV)(9xpXb3_NZbut<{h2!{FsW?_R1notb zIB1O)Pn)zySzgBl_s|*nAQUP!GWq6baJ2SRLZFA#dH0RK==Nlmql&ZS!TR$0p!;KN zf$jFDo;sHe)=a_>QrXrQcAlg0SF_dyIH5mXC(W*q%kbgaq*kN5=BRI~(!@!sxL zb3`IYwP!()cG_n_T+KNLMeMB^^o!uJ>t2f};Z${=K|w1Ou(UT_rx6%cAoR?qmIZzb zzHr=HV01PXzB)Im)L+rPwLj)YvDOOwRUZGbgIzGm;b3?XTWgH!`^|eiLDdAePp@WA zzejA(dwt86^IGW*S>e8&bRB$XA&06;-KH?qVKTzLb$*=(L#zjxEseM+Yxn7ej7Bx# z3u63x0jdcl6obi(pRAvSNl%`nsERyQ{Z#7i9Ww z-oKulpR$#)k%^~h8DQ+nP7unBAv2h!k$dK~J>CCnImX#JrnU-&?Zxj^Ef^5z51%q2I0S;VM%3)r+jq_;S*l-@jLNbbeh0 zhxYNa(TXB%j>y;PuwQ;;pC&c+U|&OlohF)}>ESbl5N_VNXz|@u#a5!GA@T8FcLD9Yq1A+Vu?2|hEO-T zQ;^mU_ab{ZQNzMJ-*`J?cR0~+WJ`bN-j1=;*+|#?^ipYOI)qV345M;^Fl=W>I^ziz zNnacVkbN6Y$7p;r&C9cS`e&lw4ZqEn;2(+h6d%fzRO%_hFZAj*zyvZ0_?XzDRNDHF znHHNW)wyM@;=1&)G*aY2poi^>J{g=J%B>8nc{i`ticWL&9-ICenj}?jMVBF(!%Qi2 zAA&*Pj_sY+L6o7G+0Wy*$`~5nQn(_vp3=ms{rRIb&m4M{if|Hc@t=yW3Y-(t5Oe-_#>CmyWz>RmD7PF=BxWnxHwM7zPM~O5@iLSPadHfsEHv#5Fc5ZnuFv0 zpp?EX)X?plOjKE+HJy|%n~5;=Q3~bWA@Mh&HJ7YaZo3#0LN4$?{V=;W;QLY!R|X?B zlzf5a>Ge`#;z)&HHfhiF#Z^vMr%s*CML~=rBw5SGn*1s4vsuJWTsNrI`DYF>Jz^Gi zAA|`|yD(sY-N%Q65Ir2tmHIR^JA5|=VrzpP|AfQgLMGB&-PA6;sBbAHC{!vT zwGc-=hp`_oX(lM$;%hgPEg~V97e`IG)XK_@gtN!L#0q&!Q5ZJQO0kXo^^(fbNZBBy29u_ayU|9L-kihZC}5i3Ub~OCt7ctMIbyIhxI~l|p!f77*KU zx0J`|ZG%dk4k3;Ev-h)_or9utc38cRox=$@33909yV{W@ z-|s{Wr7xWYASdy7atB7@MwtzucD$QJva$6T#*n?zbsz49PpWc;Woiv=mHydUAJBpS z%y1huht;xI5s4Tx+^n^%Z6Rz}K=_^#N6Fda1dfyOQ-t)2WXPh|XXm+YGUscuQo;8y z%mi?Vc@QB*T>dLZ$yVxYj${(Z#E~PIp54A@L0u=Sg|9z4<1Xm-q*JOK-W_J)@)#gI zJE%jNRiW1<&%gQr&3uU}Xz)gnk_wB&-9Ih98|RB!-f7p;H*(G*3o3U{YmVq9XP@vPRU}hq-iq){%F5e5MvshPb}D#2zlWJNe8>HLfDsj}!L-|vf@Ne0BLs29z^FoptI{S_b^K=n^^hP>ccasvPQ}CtB75?~D=6(Dg z)Uiek8;sQHcwb?fX492vRnA#4H4`B~cKl;J@)&)K9T)B`l7=A_YjK`PZo z9GdZr;Ed=eD=^EJm$P=5x$WO_BZPsI{_Z-u0Rbb$)|Ce(Cid{lsXF={`ouGlhbQSj z5Iem;BQ}0s8zcV!PoBX_tK{u(sFld~d&VFFval90#su}yz{!q5`nbQ&%1w@dV9qet%#B-eEj zW0DdH^<$EZyrIfcHis1H)=aXUKYKFymx;mm`y%Z9**?+xGJ_6|#$1N@ye6ECeau1| zT39}d3|&{X8Rxg@!11N3|9hBVO(LS9cewF2;`_cNRY31T^RFZNvkkz9q_)B!hye^Pf3!)_KdT5nq+ogsHGw^kw(R`*l_7 zQvn`Vs@?SbvJ%wl?bXS;pI(_J$4VragS2LUJiQ5%TLl3A&=hEVRower^1R`Y3)O{Rf7ez@~`$hi7t(jN!^a5d}|{!$c>ZT4g5 zOzIPFP37F#@o~G?!-ZGLzo|pU=So6;R+TD615`#5^;S(;3ZTK~|;q|@-0DfU%79pxYT4A%#f*X^7D z*u!sjtw0A?Ux#f$mul61uY2*x^}w}(#MFT~B7WDxgn|8M|8O0ZSb)T+JAB=43m_>R zr$2H8q2+p;-afg}>?hTD)%2%jTd37ZJui+7hz}kVfD_`Hwgal`V0_gY?GLq&GRQ|O z=PhM6S65lD9ni7g-HSsQdl#*+090AdvY+rK<)r~ zAnWH=_*$7nPTSqhh1!6z6C#kl@39y#yUJ$)+tO9A-cw5GDX3koz@%x9M~2B(WgLBQc~)!S&%c{1E2(I z14d3re+dhY1Ly#$0|h=$VEf%3SG%OTzO)f)#KV8tPT$89#i%;2GszVn6i$-67 zbhHUdJ*-|cZGS-40~ z-LKXX*X=$4@QY&O%9q{bFu`#^rjrzc-{$7RYdlg&Q1sFT;N1;@`v6||MVcs$kE#a9 zE}c*=fF|hImVd<7ac$OJpa=YeH<+$6UO!edw~$d5E+hER<6H`Hb$LW?K2~TTz&d(K zv2Y0nHn=@_xM!w&Q*OzmwKHB~!eBGrZ{URdd*(4Py$?5D52#zCc`p**C=YWy_DW7c z(OGO6PvNWie7F2a9g=JffW&mi>&sG`Tpli64tw1j%?X2s8G)*vD7Y-&wDVteKLCuG z;-AuzPl{RJ`6fG-$aIP1S(M z>rkl1!wV3M#@V+22dvRQ$*mM=Ux10^o^gtrt)Se(#}l5}{l^ z=R*kbuxA{JM#^g**QZ+&Z8AX53ycCCX1+~9Hec;&B_8iT1^s1Ij) z55|0O{%FVbhnGi2gX#SE0G+6c79Ri`MyeFZ7bO(X{@zMa1@LIJF}?BIZe~C8Yv^`z zVc(8Mw*vf*rPha=)-`f)Vx~vwR`dW)0yIsQw*v`7#g=iS4UQIod15uPa%NW`W{qWb zk6I)XRa;v-1fLKi=;c{$HT$fst?lyeVzKhWo~Du0`yyb?v&g=RP>qycOI9tHUUOAc zL_1y|D$zvAm;W@Q{xmQkpeP}Smg8}QM^Pi7sqD|y8@=fHNE?GhtJlK!_@?>u7w`1f z`s3AGJ)6zz`&p1;W5~u~9&S$*7&NPUY54HiP4%@$#LF1sW;(TMt<@MsRm=snAPxIF zf>0yK#D*MLkviNX`#zEfRSqV)1{B7p(mo!=U^GND6@X3Y2e7VigW3z@Ly$UHGPEcl z8A28TAQz|}qvgqHhF}$UZKr(>rZe-rhpR{j4m3)~6BHD5zdM_h7+`k1dreQxPx;v4 z@CJ#H?8gSD;p7*vK0ZEDiAan0Mo67oA;v0;x*0wxx20{S+Xqj+qV@_VC!zre@Tz}WfU zkN=k^z%mDsSM+9i)MVfU3bc&{{|UEPeg$cWK@2P((V-NMql|}JNzZ`jILQB8 zCJxbm zGy}ejdBqE)SZ-+qklBg?2!^x4s47$cFYRO>$Yh;feeA34q-6I!Mh$o+f)dj$PII+6d=iLHb;h1hRO zIx#bWqz4-C0e2E8;Yv&HqUP@`vdf(xA*7LxNCTP$gA0WIJFU)tXbnk$kY+eCcm-^n zk39D80;P|N0LJ6hg~;#Lxu=h-m)co%lzaYLpXWiioNl5cVAH)l!+(oQMRk9FzrF}e z8nD}9M<2Oijls;!Y-FtBYFGN_7d1pASy@?+xY0)!1(p!q)V?G_IU_8xp$bRKT|=!n zKt>F3sq%mgY7j$0Ea=$@tTI)-FNv23(F%72fmd@YrYO;!7^J{jjIEWD>JPO4e}qdq z8wr;M;_-L_zNf0*n{^eKUaz}Jujlx7qJcm!zt>N?q!fVa>?{aHJBK2rCH&2dzr>{hgxzA!@HjFtTA|$#R26{LX@16o z)WMzT4~K@CK_)VV=%4%cUy?y#D{Jpl$Pu2KBo;iDBYOP+Y-4+x-xAc(DHuX3B+E_y zA3v%i0eHd~c;D~ZSdJ775zsJgLw=Kj{V!6Oq_mrb1Soew8+aLIF!;IH@98~mwtqAV z4rQgQ50KRk3I?z-Wu?g8e)EQW-Kz6AE%bl)1)Tgh7;}hX+jq7%=T_r;FOXKWwE=DF z=Z|cT!FW{J{|ItB0P&Mnv*b<%#&wBxq5F4Di-a3Agc(?%XrMCz3vkU8AIqufUs55vsRC3&1-2#}D+5{v)( zN|tR^#qH13bRYdX5Y_<_`)85*q80uQTld@D`PXO^DrtX5`@enl_g9zy@hhX{?LRdB zU%nzZgEBZk4V3;5>nVok{4ew6{)0k-nm_ZB{-5&-#}}JM^lui#rP>0~qTrK%$+xYa z`uU%sW)!`h;gk%V?KbF`(glO;C)0oB+?q6b1rKa|U*4Vm!tJ#7@m3N;?_U$Ila_53 z0=m{9h7*f~7XR*~BWaIo^2ObypT1_Q_7L(wkref z8vrDHdcHp&2L3Ec2ujCAh{lpVMIO9l^V4uo5`m0ya4&~ieYk6V2m<~k{!S|)uY=H+ zcSDM;o*v_D@sAoP4rV(~sjB?VtbcK*TrF z(-Z`kC1FsZ%O_+r! zX2;8QUK*|G?TP4vW^-giqCYE*B-428C2Hnmx3VCl>&13ALrYVVc2rkpQFc03aTr3} z!!{I%R*qJBoR78lkXKa>9U#g0kK*WVAR!?+jY&IL1~*a|G^+IH>TI1a53|gtlVeV% z5u$G{4)D8zX5bY-_~g;;;oAZpZinE*t>4MU@XEve;zO8BA`|d;a;K!$l2p^B*Jgl* z3TMmIZFZGPO@@v896`5vp|w(9O&Jtnn6dM9uSRR`1p4U*!pikN8lSAo+{@{CzG{LzO2 zIHQVtXe8sPd1j=1X3m2qv_kinE2wV=lSZ~zUn%fWl9ELPL`5dRVql8&nzTRMA3PWW zcF8J9FH<1MyyHxN{3f5Sx7>`Z;%>Wpy*$xJlYR`wG0kdFc5 z>D4fNJ6g!+jSHZo`1z3S_fs_x&~)

PS)Dq-g?!xt#4TI7Nt>C7iQ&(>8vEK#6V7 z@SFZ1DY26eJ?*pyIOlb)XLR=`84taZAZROqnp>rWK^B9gln^z#$fVa&s+pf(?fx2S z6@>aWNAtqQ`{Fu#oEAMycIiNz0lJ&WZj{@3&0T5~LHuB$S%pzgMVU{}8~1R%oi2Pw z6}HRWu@W;70WbIadQ^&DRNDb{2D$vWjkBN3j^7CXA>yiRPUTc24%@TB@PsGt3=a5D zmcP(7wfX3|W&;7+oH1gb2$K5R?mcOeXr$V>O%qM}0UJm7Q21*M@(-JRbUrmt);k?b z+@BqFpT0XH3Iku(>KvkapUhh8)-B3b>rgR)bfk7dU+f~O)rq*Cb3LlKsiy`RbENVe z>D#gX$p+`49X1@%Pedx8>BVMhSqRc6=-&~$-Uss!3ktqmHiKGSA4=tH7N{~Aj$HJ- z8RA9-Q8f2`DX7b-N+SRv{Z&mPnM&>v&RgVnIKmRTcW1I9;)|13x|ms0!P_d(e2UER zf=TOZDG02{MPlRO6>EOak8`euU~TGb3-RgAvFA&!{S}X_7PQBvKv3LyxQynzbJM1k zfP7AtOB+7%MSdkB1&4*Nt-@!4_2BHdCpdQaJ$D_^Z4;q8mvGRn)f=)-WnZyR-G$4# z+Xsm!eI?)An?k$xLP^T}?tWgr?hfWpJks)TA$Cw>J^MiB$RjQ690T#OzgoNAPUDk_ z83)5w5j60d&&0z``h=i+hcxN^hj{>M>%ZHeoSdZ0xw4i|}3*3lvvr&AiPOPeOuS7#xaq2al%N2J|h`Tx+vB7(w!#5+S1Yj7BZ+Q z5rn69Nh9Jej|Z5C)(Wu9JSl3bz@$J-j@=-v`g`TpTWOt!yz-Zip*xeM_JG%1Ew9To zpC6!`uYu6ri^oE&39kq~yAHh=4C9;9wup=40k-SJ9@aczdDFF%!oCTV9X+ii#12Hu ztr#3l4HQ#0HL-`9$*d`)$1*Ub#F$Ecn(F$I4||w9^}-;iM4K{LDB%9Wy5-AP$^+&h z$uD!7R~}GJCGz5>7Qx?kf=KJ^H&DtMO(Rc_O}2|=JCA0a!o5&kutKG-<|ARGu*&7i z3>E@^>7bRFWOO2y7;ruH*SUY<(f=+uo)0(y%B2wX9ygqJ8{e%25_#;`J3%yim7n4< zQ#1OcGBY!C$}JnW$hz?LdV?FEu6H=G;KH_;bO{lN^XrON{-_fWQtd2&11oQ!7@bT_*?y(JojK z5s}qqkV9H#7!@G}p*@bog2%wI<93|+^v?GqQ&8n<^NuHE@pn!@5{{I@Q%WW)xnI=` zV=K;i|BmuQr z)1$WsHCH!=2e4O^YDmsixs_LUU?wWw;rLuk>$f4&;G6?U)MN+GL7qfGd>dZ5LYLpZ zUK6MtEH19J9)#%6pIPE)l-k|xa%<-5b3ro-q#^>Vk?eqaOU5_l+M!zb3n$Q06e#Dz z7DN--CC7-?T_f^Q4fXhl>qA|0^(UY8FUpc!gvc&(GBS!xq+Ho`&gVQ2P%*FZleDF& z^-y`^h%PE_eS_m6MzRlo8Lb=875m!=!|G||J6{>Cn$5a2U{DDO7`i1|R{a`juNW3+ zY!;No)nyQktv+_jhL8s7Ab<_e5jvA(!Gis|LYMIX8%*W$_;^jm$c9&Cn%DKzbewHU zXQdKqbtA)z6I?qj1?d1P|LD%8@sc;aF_dBT;hzBNwK)&S8E`{pxB8il%VM-6iU-~r zy{IgDv1xPXOSevh%fWh*{vzTI?slC11Fr0KrsMIVIEWE*nGbtxsuzmwid^g-A7WLI z;o9+i1c_@$S%m1Q*k68uVS3$+&e`7$Wu@hP&99qtDS%j*Y2`>nB-v8~vx(CkO#+TI zq;r41acHfCB86F}Ay@NRqtRp1X8wJZ@xVI)T6|7RmVGQRtxf|Frd?-youaL{?O5=d z2sD%T79#>BN%;kH3xyZ6U+VDB>BROAUUZXd|)!9 z6mtKudNEZmp(4j=t+;&rbu=z!<_pBgcnbN0ju@$sX1Q`X zWe!OZj8vz*!vLL)o)(;6BricUrr#3ndg}Qf%D}nVlg7oWOqc*tmdj*i?eFMJ9C-=% z7h5a-X1i2N)koafCdzdz=13o@9so%hYbFbeiP9p_!iZ17+fw8Pi3&1?$cn2|&pT<= z_svL_;t?vHx_n^@f`B0Gx|=*8Jt zd|;zTUjzag3`dLAN|M}ilW0})3lsYGpMm5e4zn(MhW}vl3%6cqC!iucrHpp-)UaiZ z_W=mJD#$E`^}t65H*dyjSx*%lOFS`I|Ekz#4kw$yG_@Zm=@L{U{CsfWV12V*>%K9# z5k9(b1y@rweOR0E^S6UN@lX`TaM2(SLF%|Y)@@H!CQEXr3bc8Q1Bh#|t_#q<8!uM-o4uAw z=W`jP_BIlL2Pt(I08mpu$f#Le_?v@#kWga55%*+N=vhNi0$u@q6;s(>pK&8Rj7Ktu zg6?5udq|)ppJKCdUDjN=8YbpouOe4e=BZd;-9a@yp;{`=V+Ep?%`K0oHpteg>7FsD>oYk4Z>v<3vr&cbmA6!Thr^-$ zYOlc@I*$ibCZ677s?wS11;9Z?g4sx?3m6zMeX< z)LKhiafeKnI+0eVc)~?7GW#7$a}P!2pL*=~o*mh)an)c4a-a;*`Ful=sLodPmU4I`JmqQC41s;c z^(zuejADD;kbQX1IDw?wXiW>km~K+&%sb|AK;KnP=o+EyM`0$PZE-Qg$jZWC%DS<) zN=Q%jfdKXEL}}q%c1R2Y0)~u9Rp5ySHsR2QnEEepdkdOG^<<{+ja1osKu=hBQ!)Df zs6SwV9@g=@d!yUQGaN;Yc6ZOSKVRpIxG}smH*H^c&OXuY>Flm{JFAh#u{I|@=eDzO zJaV{1DsQ9ZtJm!rw!f#9W!B*eSGtH*HsdI~_)+Dv^MLJ~+kplYhYQQEp?Sp6ofVAz zpra`&Y-YU{_QXcuQ@&>WQ`8Yjz+)(IIja3w2@sQheYTT2perp_5)Z0h6tGE@jc!NQ z7id0C7)}L#f}3F|JnCR=QYt~+CN{e9X6x0DA&De2{aCY(%8gfCMaqRuf`RTdk>e%w z&UDRcT0-br+2ixpcR^g;K?r>*bXuv4x!V>GY2Dh1-Ql zu}pHLBglKI4ZL#-m4rtdFm|4)u`v0wUt~_!hlY#~Ks}GzfQeJ+1_B!V6Wvc~LDrBw znJN}yA;<*`d73LhpDupAA4ayZwOu2DekNtH&uT~r*-&acLgh@oKQkMRkBCM-!u)Gr&!o7# z?nu4`cYz#k%*)B!b0kC(Q*_laZewuIXAN@ZVZ?K>-`)NxWL5E|Flo8je`)Vh{cZ@) zg?TP%vr+xbQP{a>e5b=2`fVW#6f9q6kN46M20JI}iVD>lJX%NCr%fHzi;!b-PLUMl zOX~xRE*5bN5D4q@VMk4KDE090cwDmIrDW02!jK|-@r6Ihm9%v^x=X;7JDUsJT{lb1 z1H_1O49}J=4f-xnjYfH3YiQ~6lAj3LbaY*~2=8np+YwE3tRfVHXLM@oW1e_}oyFdzAPg^Ugi3H$F7G>W`KgGnbi8XW{s86Yy9@Wr;v;Ne&* zjYv(|`mI^?mksMmO{*X-aM7yZ>5Pcq`>~F91-ak_)pmsl2{W77z>3U)fEn_Mm2Lch zksqyX@yI!fT(z$%pvs6FtP$nfnqjhS_X@nHmfm)oDrG(hk1u{|RPZ%-Zh(%Eqc=~^ z?(?qVPy98A7{BC!ifV8^Nf|9qTjyV0`0^s1!86$YKdD9YFeB^VUPY+<3gNo%Y*x@_ z`faBp2TkAno9Ri`!2;^B#e^8&R1_`au`qS6a9#$J%UqHNLNrDDAzT=~Nfr8YUry>2 zhPw6lk?DtFu}V7Ab@G%XjziDuR#as)q7EmcY%eB|=juFgqC`6|>wU_4dfuR-+wE}f z^SeYu$y@OunDiw1=g{(wrJS&Q%!`CX&x%5~J9%oo=;ebded_1YT1m*J(T4>29LlSU zU$RHUi+CUOBqQqCxUgC29(q~#aI2G@ zur=B*anZFnf2XL>r~-KQSm^e<#4#Gn@181uGWb)pqSPBCMhV8wbIdnr$D6e2e02d{ zsO25ukUqT`Gh^SzzSXut`4@hrT$NS~V5b@V0AKF-!C z?3V94V|VNbFUDh43u-TUE+p-D7~#tY)+_3#pKnog+zh@R$rKtn+rwK>JBSUDwY**& zq6g3nyXT1K0%)xoW8w_Ye+(I>>WmmIe{ey}@Fl7(c&&E&ZY1kCF4Zx#lv}72q08dVPS-E|prq zrx~dpa#Aras7+jjl2aUQjw%;kyJ#C*seK9Me@A*JZB6qYfbv&i}rR$%f z;+~T2TzydF3dL`Ir`+{gf3x3US$rcMX~#xh7&}jBT+O&gD=l5U^UobbJj%n^k&v*aT5Q;FEMctm!aBS0P3z`G!<|p}!!rC!#Oj zGVo;ez)Q%HJBl)pi@K7*8H!3ac&Jkv-*#xd>c`5~Vvf=aNo|>(e)Oj3vTem(Roa1U z<_B-P{B+xH)9B`#*V>vhcvq?$R3RbJ4bV?-&oHiWZdj^flyx;E7$ol+4dsIOeqe?uwcQVaF;-k;FjR-4nc#vOK^AhZ|6OIyU*z!_je5#ReQ~~ z=Gu=<%!|gaI?wjRsxS88od@)rj!Ty>OTX}ajww&eKgB(352#6jgR^@st0~!?3#9r- z`h^QyUYq{c34C!eXBlm;jY6QP!k|Uru_yWCv~zQN_?rf#5)!Z!f%Cd+BVTMZi>psO zj0hF3!?Et}wg%!rh%yCv_Ld$E*e~+DpPoyl?5g}GNn5pA^NJ8K3_Ur^QZ(Ftsia%i z+3?3yskF^gWK>*SYbzfTYavlu(!C|McO^s0a%a);9?>eWC0ppS6qe2Y@`b~wNe zZ@WhRWL}rdX@ML4&rMvbs=&$0=EeWAsSo0e%qoumHD2#;8=T@&;mhFWsUV|DT5Xl z66;YQ?OwM-Cb|NJ#^7^{sO*=27g);do zo#;ooWpXyYAuf(JGrfjqB*VQ-8AVbkS^(!o)UMZF_s=G#R?8PN<1Etx`PrC->C#~v zLt^>bV?oV$sUQF#tUe<1Q*_Tt{gczd?*WzUt=@B-PM*dSCvp!9@mz3=^-3B2;XD&` z=QV@SYFkIw!quD*yB&ahZaNKV6VD# z2e8*JU!-Z?MR^FRie4pDf&UA7zqUGO7c42@k<{~(K`DWfoXGt=pw)Dv=9ehDy(%)v zlD%{;#_Y_K8HK#t(Mb#E4JDaaO#$bCSdGs?h00?P{e|I| zcw#QG{>?7_)%!d7ow)j=dkn~sbhsSvtKV1#oPo_Bj%ks2>Sd_^bu z@fa;N3@({e4G@R}9UY#yPn1g5&*_iyOlN2&fS_}Wf9;<)Msd#3gJT~F=U$g@i=AjK z(Z&JTDzi_}Qpd-kW!g1E$Wm~m8sEcY?OCu$N>n;+sh4J8!s8ntL5m|IxdWrwNz2&l z=^ci23iQNv@{M2s zACGoKwk~g=yoVvX&s$=T>-|9rjU$m(caD8Wtv#`TO$O!ahsx%Z{vu}`qY7@~1SDX9 z{u}0Bt_s;rGY?+)dV3gIxVrUk9z7M;f3Y6Z1Wj^ZN+v(kQf$TjGesHHkTQ%m+mE5M zLmZy~_vNSQ(;{wNOgCbQD2C9de^A<;;M52v(L80-rp`?luI35f!gel)>n5JUhS4?> zL`*U)(uzvyw-P=(nzk4LC{6eysPuauwVJtHM48r-BUrBuibV+a`3<>CaUOX$Z6;l2y#SC-_{h;s6^nuL}7 zz@8N8YHP8!<-)KtU(fq8^m;bC`hT1zay}^I{t`k&kO^>Nwwcy+e7~1PI>o&# z%sO_NR@|>~_#Zg}hmXo$W2h_x=X8Y)buh`61ovyl$BwBdQ_dy$`ncPl*?;|X%WV3a zh=55*XQ8}gsf)?bDHgIHg{`Vh4u!=a8jZiUal00Rzgmp;T!b|cAA;Yik>3j%^!42+)crYMIGfF&FhzL& z*}>jJzfC0pB3GE}KhgczTI?=sd7BRdn3 zOIL!gx1nWe-xp9P9p!>)*2^)?&>EUw6`Gw^Lah;Pj$gYH<2f0rs^XodRNmqK} z^aN*i+}0XZAe?Y!*(=zie8YWqffSs6U9X*Yy@Xr50?VHKW}@I?imW5LvZP#TnVo7G-H15FZdJh z+6hNZi!g>oQ58ab-Gx}B0e)E;1pQr*8E{3{VNIWnaZM^5tds|8b1a&nqREeQP&0=Z z3JNMiB`H?-xC&l&qy)eSPXCH|0M`d#^&(&orr2J0;q|^{8q4YonjhoNT1k)<>~s>g zG#)eTru25f#YzDLFgW{|9QmQ9h+~=-06BFS%)!_sT{V8=@r{!uCNk*#d6AXt$}=MH8Q!s6 zSUhgIbAfv(Q3Gs#fHhB5at?b1xrC&)|ltO6P^Is_~iiIoVT zU@-l2by)g35WY{C_8}*o$n86A?G16OGx$Mm6Z#>k9@qY1=B{!SnCFw;N4RXdcGw*rgNE1Ew zo&nPcZ>$+~qUE0XL;R@D_KIaK8W%Uf8TM~p!g%vws9qs9!GTu-2{u4#g7WA9lWcrW z>?r}6Ac>!4GOP0e2bGUKEgAve6N2@`O_Qyy?;>J0UT*SFC)(OCN_Vzv&QI2-XR z>b=5btsE&L^&WI%OF?3>QL$qBunPvt`Cej@^33syc6o+i^cgLFE) zko)f^P&IBQaC%H={n|c?v}|~#j#VtPd9OG`CyFB>WI!)kkQB}YoW@g699rq%Ek~RI zIu01d<@?35G|KyiUkr?9_>B->PkCHEF?y**%u9Ep`BO;pz0zEACpV11mhX7?xg0Ei zN(w1b%z^N1B|LGqklq~u{eGqP(-uH9atsVs*$F=;%RB!yec!4SpRqUIpPp`5WHGLMW zLVTWPTu(t-S~;nH{jM%jKnzKND)rsyeNY~}q#}$xzd5TaPRONs;MkZ`5|%L}MG|7| zk-2&jgE~y+{li0K*ZMlC6!LbcbCoY7 zF8sMcRqr0(+F*{j9eYcFX|VKUttXD6 z=i(cV=nSD1yyQSO*PW#f>$Z+O!v#-Dvm#M+YCRNDgWa#}tAYI5p*^G)Ec`Dcmm1h~ zqT>FI-@$;M0CJ^rukz1#C-Anhwc2|2=6Mj@>c-jPG-Cb zCLAJ08#zYfiMFd$)ZMQB`@$T-UrTvNdyqVRY#jEax#&&=t%;Bd4=loeEcm7>7$1Y+ z7Uk}ArDuBLvj8g9Jobz7vvQ-u=IzU#ht-G1s1yixvdvNKm3ERqnaUn5LiudvZ}%^! z>m4!PV%QUJJoZR-u$664+Rp?%wVR0<YSzZJ?&3z@yx&hfxdcC zz@T(Vr(fW2?p*FdgnpCX(k(+U8dTepAr|I^Gp2))=uc)W-%WM;0r7iLA!}F?X!YiH z>~Ll!y!^)MF#DGcy^bm6G(}&X00qH*w&{BB(>rqO+MK2R_9{cga43Mu{%?0Q z6@eNiS!h-10{#kWK%b8pZ-xk84|`3**voZBI`)a!fu3vhG zGIaPwG@lOzMwa|g9kq_-XZon|CzPI}S&VZ{xjhjAIAhzC_fpLkUIds_#LSw21Oz7% z(zDaPGCx#R!CL1{`C;Hi@dKoo*=uizs#Q`pq$6v^ukc-CvZ$anvOpGO`Fi4z*_2ixhpSl}KGF!RJUFo?bvWK+69eyy`*L zoamlDDaLM7lrtKj&G;^@?>i%`Y{i=YZ(#E>h<2<+$5qCedBt}=zIs07X|@_yz2%k@ zM{Z!lgkuUBWf_H&W$}_`ozD_q4aw_>L6j4{a^h+7cqWU*T<58A4h{F&W6C| z*&hP#%%@C`zP1G6sjipdNVo$j&xyJz&|qVl%w^C1a=o0rO*rxoKi|oiF`1OCG2sjM zTTrnwLVTPKUcMow%u`W-;|}J6%LC6$3eo$P5{(`Ia>+QWr+!MdxO})me6g?n|3vt@fBRSHu@0mVieGjdxosrV1h}%)P|a-u=q3Wh08{KeBx~-&@H(e*q#@ z*l%S`e?ffG9v_$? z9Wd#g$i=^SiA!yE#g1L^IPr<=8V4Bp{|+{YH}6#kTf6*XZCr`RYm2Y`h|6g+kGskb zn`^Xaa9)jpA%2)}f9n`U{oVr05G=Xc)%N&-xrGDyd`K5TE4@Kt-XZf`D-&ywK?HNQ zfHnA6^R@-u2?EBJ$E#rV)&r~(|2DF9IJcp9&hq@IT7FUSd=%XG4!Rg9=RQ3{KnA;6 z89--IjTCkbv+_0!+Mg(=ZE182J={H!!z7aE^D;e}UdWe*-1NIZfE33H_`GuOP3`yg z;vU5`kY-jbL|PWnb)ZI#+b&;6cfG3i;Ff^H3nHVU{`ZWAglJtP3_im3$o&hZsW-%qO zq+!nt;eo{ic~|R4+Y#Z1UynA$Q2iWNz0&r0>kwf{lAc_4M3( zW#ycN;i0`@dyr8`sqiLEW#1}&b`n&XV{bOrP6oiizG;;G3GAaIVH zKJoMMe%h$zJ4E|2%T#YQ_FmdY+%eXZK&&B8b{rlO6fI)5u z(IJ9K(^=FD##?U92irV1#6!lU2v&bWpeljzY=m!r*?vb7h$iKA5{?Uxgu?2cuc8)Z zLdO6_w}2Fit}7urBj{rHF@d>4Ld4=#zKk32)uIdtiSgq;F}Xvx9KnP$9d*$ zt3<>%poG3dsK@H1y2ypN0D`vuC)6VW6Hwc>)C6KdXEECZyE+GnCbsf06&3;C@bX={ zfce+fY}@$uqb~!$#5YIvU@SOJqQt>IE3qs;wY2N}_G?)yt}MHvER^E4*Hi%O)r$MX0jxTO}%cOd#1r@#M3gCV0jf|T=msyt|1kE2Z?jp!)H1W}p zcqu<@K+%Z`!vE(M76$`Vwy8u$vY1_|aR#`C=Jy6kI4o#O9SbPl%)b2uK*xmHrskiE z9fSCy^lTvKi<`QCl!!s*&)Qu`4iY$4!jhO-WT&IoDFec-sL)UcNd=yFN@qV;dHnny z-s$x{><=K(jHykY0$yG00h5UxJK6Cg+6ciKW`G4UnZhUgpPU4@}$VZvV_E$0&~qy+PSPr)uTvjWzz zM!?AWPpr&;oWmvX;Vz~Sl__~&o8e`-6kD;MuCJzHWO2sgUZ9NnT`Doi9Q|Z8Y9THp z)_uo(s_%4X)w8TM=|#WXNunh}{9Y!7&s@lO?Kjsx zCxd{@PT=4Ayl^S-ugd=y75qO?7!{?q3%G88hBnzQx`R0S^5+BRlyAr^2>YW?8bm~I zM~B~~0pSoioBrf)ol^^n0*O+QlL~gCs8_U~T8Y$w;$TS<(#X6^UMH(Pe#G@zl0^#} zQy~}{6#qI|O7wt9GOKYR*>kPGSMLAb`u~0iAYkl#26?xASS2J-D$W1)E)E&=whd$+ zY|#6a0tkl$xo95iMws(Y6B;8r{$rR;;>;@(Mt+O0I$mlB7(H8_MU3cHayvJ${aI1A zwqJRL3yM|*r07NSdgmhtD-q8tgz(}rKAd*v$hf-VA+qc68(NE&6t%LeVplVfQ4L6;5)7OrlLD|QDE9eS(H z&!98+JIQyRc^W?;#Af7BSZOL55{Hm~^k=y+X|CDH@Q9Q@`~84`{Kf4TGva)PL9U@t z=3196op*lLG`0JnT;sNE(=HGBugb*`2eN~Nk5AnpApW4rNs4LyPoj`4hbg_NI^sD4 zH$oW6^-_2TA2iqW(AgbB;90ERS?9)s_zcfY-NfzySmHJ|?w$MYjFeY1ZA3W)r10rc zm=8^E?(c}(BjW{QB%Vj^-I-CcI~vS<9J27P>h7OLM8!E5<#C=oCyFpYjIYqkbMAY8 z9(~9_Ag`9amB}%fvxws%QX<#XqYX)nZ|5%RY06?*{{feP5i%heK(c^nq<=z-{=G?6 zW09$EEfYz#mh~9o@_a-Pibb5|7-|;*L^T5NpsVbDjK5%fnwOq+eEzJnVAX-TX;(US z_x;rT%oF7<@I(Ulk5aXTGMTtwtY7p@F5HsjcGzf<%}gJm^>N2Cm=r&^{a zlEll|UM9>+-!W~(#eOtOQ6D6`J7w#W29sg!(I(4$dU(%80L6Cshfzx`-@k)2zw|LJ zPJd}A(dmPY!Zi>3j~bi7IJ6yF1KImF3I`yP5zun}6Wi3Q0LB-`bbcis10pl507+w9 zl#aunwOTgG{#pH7m%SHC1HM8(;-m<^ipFy`&Xw0`F>!S$bTL#CV|9ML22);05bqo#T&Md7EMMFIPD?%oI&k^;-m539r?JPDMh zNIm&!!LDr41%_6IxcgK~W3uv#kw?@rM%%OMlradE0ZNiE)Y-+!W&4)HUggn`{~kma zG{jXxnNvW@k!plJl`mc@O-HAa@rhAXQaGa@J;}^F6Jf1ct^EmIx9-4^n9qqg>`gE3 zVclI)^h@T0P+GLs`PK*A2s4ieYE-4G-0F6W`cgi>9nltJ(+%aY6D?8;RYA3{H9XeX0H&OXCfp54!TSV zqO$VjJ|uyPJ;>V#9VFX$LI4wI-V4|iaG%f^_qfHB%eTDK{%F8t&ac&akFyD`2Rgu- z61^Nwl9_ekKeyK;3Z{etaXFzwg3DJVL+RAvnbtmPZkyzG+@nN!45CyNpRIiTVa(^2 z-u%S9z8%&gy1x>{n$ke3`?frvl~0PShfU6M47Upq%OvI!cdTA4j1ek|Ik==T*p9!t?XjX}C{Ji_aYL)+D zv`*h4s-ts49Ge|7o@R?UrC+Zq*%Y{V8aCB`6x8@%=C5T$K#_Q2zA2vGFWIVW06k)mwT(r#VbvkSM=Gc56EInrv?tgB)yuKzNbPPIy-s>} z#`6O>%n|@UXu;n*cLEcXWNd#UQa_33RvPJVD>BjONpr~^Qe^_hn&1M2zrTJ3Mqv6-jr-Jku89~8zR0S&-vyt;$eBoVqMg1CBT2h~Q<2TJ; z7x&keOldUsj=a+F<6Si;R_C%7l5(aP4JAnizj5QMSEU~+Z`pl3z#96=A9o8b@c9&< z{eWo7%F-`eBR%NK>&vK)m1g4!^!G-BMwGOQ-J2vSKrZuL&QA!Zz#_&}b)~d3>7bEv z?aks*STkl(&c)Ka=`JyR&OtvowEZ9!u>iV6#`+{pY!1!txIICu1Vb)5QCW4M5;*^o zF9{l8SzIRj(>Yz68%B8wVLV6*>GNXnA7py(XU>P{90;JTJBZ^b$1$-~5^ja4uV<&c zDGTEw#WziCz2^0w&xY@`r-+1Rt1@bo!G>2HOBy(9M@tF!zmh_I?Gda~5)FMs)}*1m z1+Lw)>Hxmg1xByT8q`1$>@GzI#_{|uNI6c(JjNI?eU%BQesGAHO$+HowY}W-)MmS3 z#04bgsDtCFHle@x;(@1r$mePn*I4sCMQ(XkQmB||%6W1TDJfsCjJC@90AN}4#tpaY z&2(foEtzloLuCKA2p9wm#2y8^V~nRW*lhmPrwJD#OB^yPdM75)zkdP5aA!dj2huw& zz`R^R;~N4}9Q{||nq0xzpXpvPAU@NI=Yj8x%;T`u{W4H_F}XEw?Kg z@ppm})=;b&MeduPExyn^XDb6&FeE+!I5 z(;k0KYE0mt-*_g}edIxE1x+DNO1Zs@>Lgt!0& z4x~I(I(cYuPo9i0aYh10#cV&SJGbYswYHTKQM(lX3T?E~8KX}gch0}EG5O2;;<4H73Un$|D}0x& zjPoli@`;=To%ZMxkd=V))x=cjr9s!-JJREg69v=bYMaGjz>|3OZ<1srBW$?jbxd(l zmh^?AhK<^OYknkklH&fw<;o|RJGf* zCWg3X%v!i% zJggWayo6}Xp6s9MKlfa%Za!2mSk9l~SMMfY^kdxIL;Q3vjesI(iw+z-4fYFlZ>sl3 zgM*(1``^yY34MZLp4vW_8ReE>b00rZN!8zhK7Hnb59%lh{PmqVF2%#4&gScMCx0N^ z_^;R_6EsG68j5%C`hRWMtL5v0s;BUuo-jGr1`V1RItV$DTaQV77=?qMWB8!;bDhbJ zFVY(BnUM(Z%~`-mG2Z0&aT>}MGuM#h{^Q+U+DU4II#%u3grw#ys*o|>uv>8&j~f^2Mf30LMqB2OIKT-fKJaO?x& zUmuYRsqnN9Y4 z9I{*XHodspMXvtwO9l@7fnyLYA|8;$>Mtqv0jU6MDsR;5sXgsF7q8kx-I0^wC=3O3 zmyqghCI-WK6!n{km3-zDeG3)sdFuV{+QdKRaYxQSH;X&?vlT76coCUwx)?t3<>t`W z>E19ZnG1zQg#+iyIhJ%O0`_(!Qf1i%qZ;qZrJn=cG$;*XjC zITX9W%u@a?iaY5FFg#;WD=n=QO?aWq^|3qE)JlqogsjUnO5Nmr_+wU~FYlH(Gia2@ z((+t2){Gl?o!iu8g5|}-fZU0aEVS5ncAQ863!RpGwjI-C{`Lz4eV)~sq>bp2iPyFa zT}pT%>_EsiUzTQ?`^Rta`<~8M2qylQB?BJLw)Wszv*jG;j_ZlE@pd>H7ah?f_M?wy zpQSRt#|f{Rj;U2H4)hr(HZig)n_T>{q_cHegH7z!hulti7NM;(a*BK!o@N0Za9h01bSck|pIO$XBG%(LPJ zc6mXi)6g1ULhviMs(PlkAI(&roAx|ML|g&>DuKQNM7EiUUfMBat_JRv)bS#I)iOGG zEWG!2RoZ6b9w}e11a&hKm7uLg4D427#wwk>U+_uRbKUXlvKSs$Ik)oXzrZD%#G}cH zK7ezKbhjVzHD?gMY2x}3`9Pn;{%@+(WF9XrNUU2{&=-b?Xb=y^4M00AV|2sMSn9|9 z`n~abx!}zA7xsIG`N=hi%6f&m0Qep9tQ+@h7Dno&{B4m+bWd7AA@HO<1Ky<*!kstq zj9id;{w?StoP+SB@Y#;Bx+cHp+3=sT`GkLmP9&yYXqjBa{8EE)6qzBY$3{O%zBIXa zH=+GDQWSB;5AwOpGdYJ`c=IB4vapfCNzv*2j9+b(L!>`QQPoAOMo#x4P9x zM>KZTLzo@qjf8Yc#L0g1@z7GL|paG7~T?8A|w!37Q)nR}z z^u2BpX+)dCKO-_aCv_U?AXY(vbT;v^dy!2|fz5Qk{V?HB3sWEZnp4#|z-G&X=(pQ#w!NgBMm=p^ZpUr86_w(VaCL$=9-{QclB0 z-xOo?!T3ghR6qL;tjk5}qTx6QAs3z?K=mXVWA2k&8Ik+ym-#=U)4=Z58K+=#>3WjR^!nz*wZMh1)=0kmGuq&2z>Zm#la>Rpv@O);CPvhhw(Dqldn0 zsQ11d6oD)vOBEy{9nK=! zo83Dda$}aw`5RN6r(eCaqWNpnwE}AOr2R_=FF-_fc{c9AdHBG}n*Qy5j*ypUyx$ry z$=uigp%2H$NvatfqB-q1WqQ%I%>Dw__sb~y&N$?T1SqSdw!lz4Ofo$Y&9Z%6zf=nA z+WhVeM(H5?h!Q}6jF9-rvhHCF$R_$P0rho>lxQ03qgkWgj?CsN;b138yG zDj&kKa@W~suc?$+qPro4169lY*9BT;%U=1t67F*s_5D7{v;tlP|DsXC!p=A?`{xXP z_gw_NZM4UpFIX+WWG0d!p0cov$SgzP#nCouhwY7uX+p#xrok_kA?0T{ZiY{vnkIQX zP>)Ynn2p?VQnjW8n!C{)QegKn`Tf@;IhG$;2&&)(F@N~!(%~4kYP1Q*&iopx4XEwW zG&1vqN>4EzRE@KRweh^l(XJE%vthTb=`F(1bGJsL{nh;UOPs}MZ)M4-GN?72hNc2A zT{W0=r}{viM8BL-%w235wQ8dq?e+d5u07E=31!l#r|<_ zxy*7)td}YnTihOz+~m!Zi1r8zST*sRx1VxLAlQR9C&H0-O&t2qcJxuW()?pi+r4r$ zRDCrE(^(o)0`9#Yor5kKjwcd>pmwzoG)f&l`u#6V>`Ub&kXkB2n+w11snb zXIW`PK4bk!d_jh3IE_ACUCLJrSpl6uUDRKSkaf5`$gDr&-n>tb{SHhzpeULv6Kc6R z3^^syBd^LJozlDJp2Ize>J-;PYTvg_y|CzCxMI1nqz9rkI!;54+P!mJ(rJoLvu8^% zD}PVnDA=*00612Q0ofr#^yB7Y&J>e&3-umI z(pOK5@K>pCV8Q*sc1)_Oxg!@)H=0m1nZr(~$r@nCpkf4b#)0qXwt3)# z!hx|=q-4%Ex<3GtF7x1K^->WU14t9? zDj`p5P1v74^cDZypiI+JC!C;I-)1afR;^M_gR?|K?b(50Ksg=H_f=i6B)k!#+M`p~ zgjt;rLx~VRPriDjM8y`y5Q4@~VCEkd+|XnMs8j(y zav)moV5df3H0@BsX?nMVB(~l6B`0Y;vBiVu*-2@~$*Wi5`NdbsoPG7qb>}=ttI-6r zR62)P$x@TFgMbIu)~2(QlXTQsA+byT4EyXzt8la7vh~2x%;@8Bo2Pha&3^qR)|#IE zAuuTHC@%7Q+1VMusDbhHTgWPl*BH2w7;lZEj)n|pJNp-<3{|ImROTq$m+L)DUvA0? z>)0{l=Q=4t8J-r@&TGc0v0Y4&EnyytB`B*XN0XzZpY5G&@4p`&7)h~hXWwi7TsN`0 zTe)NhM%Miq=e3~Z$-CIRcjbI9bdPaVha1Scc+H8hzC8cz8$8oDc@k&K(33-XetveQ zHr*D-|Ltj`##BEU0A=DquOxVXMi@);Ly)JT%_`aX(nhq{T(9hpj3!7cOq^_|-H2Q| zlEQ}7fm>A2h*p@%lcAIttKst25n1cO5=Ptr9;M0xQ$f5gR4TvCvfL)M=vsDv z6PS}b=6xlg8vbQ~UxI)!94;c9J@TMdBTlVZtd0%MGjJ?A6B0pJXkCVEqmV!TxC}Cr zK=tr;qUV2N^ioIQ|3>V#WNL&5coc%7ap6rB!ll$~5qd)UPBaPM2)Fm^`br9e%K=!A zGnwuEWl9-XUaKT$w`=sWljG0q&)C5qc3s{(|FUje(qR3kbwm5eh~-g#T1|qlGbWy} z&%3OfIqF?9X~kRY7uavkqESBU#Nj6!SZ$dA3 z&sq3De1fRck1lM+(4^4gF5jrk2)|c!Kbs7h`hmZ>-I9o~;rwSj4Lsu?ewN--ep6mb z17a%a+FI%`vD0waM3g}4DgEd93=PzWOwuXwI>K0IW-qsk*)AxPIu~YAh9lm6PfkZ% zk-Jp@S9G9e^Ng6ZmWj3L>!~K;vZW|HkL1s=cJH8awgA^uf#7vg<)GPb+BLlDt2&c6 zW=<(ep=@f7sL?Kg+=)i?(Ot?k6&nRA+!1$abK&v!oLHe30Y&rEtQ4u z&$?x1ET60o+gZsBOGS3@6oXA&1~QzR$;f;6}75?sesgIL>w}Y zcAOX-4spWq^+Oen3#@?5)tWR%uYM2Q?vM4~Yp!|NniGnx# zK=rnA`w4qP^El}4@>e{2qX?uZi?&X6Fc_wB4X#f?6-fAV&*i_)cN^V%^!2kz=qRk2t9wwK$ZIq|NOozcr?)`#!V4elM`=`j6AjZrwIbnYj<)_&qewGyuC_iR(PN#s{vFaG z`q*j@ABC8Hn;+6C4sEu}6s6)N0An8V{<58a^SqJn<185O4>m@0)FC{n2prw5%u$%| zZV+j}s*sDW@$?i{iuX*#LF;Ck+C~xLhSixokX{{X0HH zO&l^$8fPqqo_-!0=H9P;anmG{a1L-W{U)tX68?s8~lNr9p~4He7>qmN8qoHlcXBH(JM^EdbF&XRUy9da(2oEBo}%)`&phDu(m1 z1Q{spb-xR{Uenc_8}Z-l?AYE*z*gx)H6?nRDv9I9e*IoW-0<#w_A>32AFf{N;bVXE z=?sGs$5~_kIv)F8$LEj@-j6qP2**U~ppf2qBoISbcPuN)(E~RjYtT(nF2Lbav-blS z08(g*?h4s0p?W6iq~twK-8k8NrsnOeJQ%^|e1~ZOTc0S42>^&y*~o>`Dm|z>u_QJD zWy&{vr=dfy8tFLrTHEhO(=CxAzy0I7&z-6D(mD#cMIQUB4ZhF%0fkKm6gDoIV@iH8 za3mB@dc_=Q;|ClG?3w|Q4Z9=yRmeP=L)CHfdwIE@W!c1rT#6d8SI;nz3H$$)HkI%s zs23cR3>(-2f!Q`}dTb*wi+)fnEB8O5@P86b-LVBO)H@Vkk@nqw{@$LNv~s^ni{uua_b0DNxp*26YJk&36ssf z{P5qhH(9y4W7ltf+q^5{yQ+9Z-8y35)-Wx*JUzrQ|k!t1iGM?E8^TZq@+WW;- z?TgLqWA08yV%6#jG_fxK_~Tujd1w7cSun(;Ryy2GwmzVzXD(@4>ow!Jtb1;e1<3Nd z#*Y3U7Fexknu4?M^9S1b>g{(~^8l$3T`q*$me3sB_t1jyos09tFNC8sWw5e76!-V} z^T`8;cN;3yhfw_~Tte5EVrCZZ#--996`_rO90C9y$n`ww%-7V{reG&A;4Xc#+ckUE zTxR|xmdF#!ajt*N-IQ6BqPo-gm@WHSDW&+Xt4chMD3P&`TO1eFsT8i9oAr^juCq|R zXK@C4cC9A?pQ5VAKxdLnCAjxlq z;&$|35HZ-$#WO!NVVwKweTo?uNz&`fHQ&eSIbl_KBh3sO{>Kp6MYAzN^=EBsJH`#|ucAu5Cz7eGX}=5A(1K) z;;SXjS)1Hsz1m;Lpf>1Za+^^oO{7cN1*&Muq*)9q`Es;)D@UXWrfGhAgy(Z?BoIND z_aG0yZYx!8Fm^;&x;uaj5y{nDGvm)ME|v*c^VdMirTGv?!OQjobNu4B5m z&UzL0N~Q_qTh}M=3wiNt9}7n4S7ul*L=J3*IS*Y?NP$^g)HH1uiSi38yT%e5|BUQz zvWnK3!rEkKB3W{>EFQZ=<8YXGR3o5%Q)je)U-tR4ob+FHROC?%;AG7UhLVIW^={$p z-7q3HrPu)Gc>GBu9ktA=f2!(xr*@XLc^l$?ezz-9_3`j-Uu(d*uL;m=|4mA>-pdV9 z>}7G~nI$UVV~rh{!^<0sm6mH3rt_E|_|+5xd(Go7P5tm{PAz)fc&?DNuw~?8X)quZ z3Mi@i(1PShgWW)A{Uu_hr6G%Gg!9R$PBvL>E?-Ke=Afu_&N_EG{gE82%1d`9^lh+a zu&igSC_r}nq3M;xJZ+g2ha_2R9bo_0wP+ELV&7f+Az8MRA>abbE9KJny^q2iHj*~F zH$CQiN|=xbZ;PLGmY)r453GLDZ4>&dm~CrbD=^?Cn*?SV$?r%E%)9hfobGz=mRT5a zR;VDwcjiN6hel(P)V^NnVY;N&7r zQTsq?0XZDVgmB;(1J1ud8tkFlXXtr*uft+Q4Cc!l5$n4H$?Khu-*j%v?zi_E$5{|7hL0KprA_Vrna|rms;I9yM&l<==~EdI5lP7fLX5wd<@uJs0?>64E(;qnT6TcK) zitMcZmg1nvYA3F54Dlofu7QUQN)Za+dro~5D|+E82#l$eRUKRLkUTvZgzg;~R9Jy}IOBgB{0gWa1uCCn)plgurX{KF#D(Quh> z3Igl8b8xU*O+T3_t$o!RhJ7+T^PT6T+X5`?k@T2#c6p)dIxHqFUI`ZLSRkPaw(-4H zWB@ml8f|D^(I#PIMwnF6`3?7a(tjNuA%m3oniHJe`RHktb*U2yN%zWu3?^NrosW?g zZN+VKWZsRLrdLy-o?(AB=0VbfQHA^^>4|~><9hBQ#}1wWW*{lhnaV(t7$JVbLTThq z0FXsu41@>Z|KiChZ{rBn%4hd^-!VpYmfjN~RvyuT6^-=ME2aXpt1DRF=rGX`oNwq> z55azU$}s0o8eFr8E0M@dIhQ?suO&5K{B$9E5|3}RnQbF>uHxvA+mvdrDUtqIeY4C9 zD?6+eDW}dXM+mt&eW?PU*wl~J!TqHvfs?(AI)YY|*q9aT7B+k*NR1f@m7h>%)$kX% z@Ji`_mKw2diTh_nrm2~)+4R=N&;mhGImLcFyCeOK^%CaGz-aewZQ(_&bE;$@j4YPV zKWw(4R`i-DLKxbIT{bjQzEEr{mW|s)HUU&o1>hE*4=nr$UAhr}nj)+!3m{ZfE=1yI z`7!}5X-~k%NO7*iYH_DS!l2Sj-{~L&=56aP8L1=(<@x!Zp#EUP1YDgQ^bUdStJu`n zi&hke~q0Ac(<=v^DtuJfg!u1ipAU6cPq5 zMVuaDJjlOy5=kB~1x<$ODjk;V?RmQsuj|yEGXK@zgZ0Gy&HPk-4DkEre2d6XBRLZC+RJ z|Hs~2MpfChYr_j96eOgT?vn0K2@#|_1f)y4Q@TSyx~03NC6yA8mXPl5{N_@hXWx52 z@6T_%W4vSRe`~?I<~8G-=bUF92aEyP5j^943vAot-S9x{BeM5)<6N``bFISpr!H?D zt(rRz_vtm!5^)yRwK8G1>UP4tI^VO{ZyFP{-oOz*MrhWAzOx|>@+A7x=?pg&L1af& zdR{TF%ALjIJu+bf>eETG^7rXGmFhz#pawA-xZs6>|1Crac5G&FO)BoiE#_==2!CM5 ziw`L6k7?$th){pJTIk?y*l2}YeTYK`C8>YRvyd_a+haiI<$aB-RY`&qc=(g!Ni8Y! zC4pQjrL1;Mfv*82nn;O*`atvDCU7T7+HZff5odW~gF2|%cWgA(aqtlQ1<6^d`N_q0 z+{Z*cMhP*cU)G_{Bsly^n*vX8izZc6UPat@)QjAop+;J1`z-nrerL`k@vurDs0)t9 z0X)Jn_j%9Unp^B2`#iCdDcEJPn7fc zjQi5tP8f6AJO!Cy2c>BPal+3M`yz^~!-H-8oG;Uj$4zUV!N$Ss)bUMnoX*XV9J970 z6A0+@*iixSO?ZWJn_nA&jd){us7NLiF6g;~QFn+2QDJvI%F1l@_UwI%jwGt-p4R!5 zyL+${*v;Ea4(3a(s&(5HkMiC=i0aa%&j6vB&+Re3G^Vv( zmIIhLLB**7R-%Z<150)_0S9S?dVtM(BTehlP2_RaMVYF2dS|w;=Sco*V*4vpO~2!w zt6@Zdb!Nffm6{5#ODK??lY@?ponqQ@nkG-|gqB~=B=IgI5KtQL5R$d`+Oa?q?no|6NwatdYJo}HO4@zw4kt5 z;`i3GOn-0u0HPln(a?n*HiW@aqL(M90D z&56G>whoPEd1x`;XY&NG+;sE`Y~+2wPEIv#oA>HG{}oGhpyGjRc3Ijc>xqOiTmg1B zHo;~f?^ewg=KBKsmm^d-E0n!`@Qp<0-6@5qH_x7}T)2_z77}~#&u|e7X%Jm6W*ltK z2QASrnWQ~6V*Alq+-2HiuhTp?hJT6eN8|L|6bn?&XK_a8@ofxhmfzV9rtTm!hv1jMfJnKd->U~shO{zeT~*RDdGSp_qgY) z*EQXNmV5IR=HwDY?Od*_tU6Lb6BbLR@frdjq%~pu4q7%0TNk)L_a?hU%?TK)2~}FN zODt_#e4AGC;10I) zzS7!CT;1U|O!}3tr_SN*cpJZ82!||}QrJe(G%7BLbI$M!R47$4YW|giC-;MMJhg)X zITrYfV zid9r!n*uj=SIW+#%_f?&L3$hXgoP@rDbEIW+z5L*|By_v5QY5bD}hAk>l2kpKh>7; zgP*El>Bu}Du57F}ll4;S>bX3KTT7|E+XKZLFJW{<94^?pBnrgmbwd*xu~6(ga7$IY z9hf+qwSgIGB^_TX=v~-f5+#X1TP6l0c$DFrMuZceHsAB1+YJ{d6u}#IrWr8hlKrx9WNa!Q2 zR*Kf#bOwsJKiBeGN|x0B_(;@Su`5MzHNig8O7Uu__s-L2a|&5%Zp9hAlWObg@ej_y zO0ph8_~07vfb0FB+Zbhq2F}t?m!!2+_UIEdTMppZM%U+2@kMAoUH|oICQjl>XPCgo zdU>^}3^={PF#vG61OZ(9(s{3ywU4UK{lLELd(nu%Hxu7Z)vvF$Z!WCtKOBq~y^L4> z{O#iMQ}z${wr|b+>Xuyyo}c(>R`dw+G>5rRgQR1JoiNTZNDMj-je&+$6FkJp9Mndz zk{I|BilSAqIedw?zyq0%$CQ!wdo#rnR_W>l7kt=aCSRfJebfAwkkdvxz;&6Il%z?~ zE0{~a{L95zX6^H^3;=VoyQx{Dg6fTaF1vgUuZfZrnrFeKKbG8m;ra%1`SIe!z6`vT zbtP-|$A!)VJAoPcpdG1RdJA|50wLmJE}M=X>}uv_Bipy~?4LQ3@C_?f?nk+%#$~4M zMfnkC2582N;fzbCd>B&hx^52BR3hVDbQ19b4L^w)xrBE|u+9(-v%|~h_i_(t*5PQ? z+4Lqj_OrNKnkWyMxmKf-yX1_nY#;UH4l+oPhoyMFg3G0qM-=E8{8Ck`RpEJ!Q=%n^ zZ^)-6A@_obwxbDyO6ytoTTnXD8o0^9R&bX9|F1fEZe4_wzt%!p--88`@}CtG@y~+1 z3FnJ3>4dwkd%mr$T?b}rmyI7tS=J;G4Ly?5Gh}5f8pJB}j;KDK%??%}9L0}FP-oPk zi7fy!D4>%OxV5=5j8$`BQHwXJ(wcGC?e)0$jX+GXY>t9AGMh!xJo&^Eq?*pql&xhi za<9dRM^!F%u9pL@ zt618vRKora7umy9{PNdEl6vi3RiP3eTTEHWBvHG+KhU8KSDIJJi8L&5ef-gCKqio? z+k`Kq=@EM2cduG!^p{CUFY5F}ju!bSJt6T!&@7EBjtPdh^WD?3qOuJwhs2!i zZmyNlte76X=8Lf=6R6X~-D3$3ILhwnE80;&R|DmhZ$}7I?Dbu0aY6BY(0CW2Y~Z(# zrk)3fhbJz|7c0j!FMK#{)GTYC-CsS)?^UbV++9-cdZ|5Mrd8UjK@D{LA7Fr4B(B+D z>=ngT%Y>o4Kc36V&ZY>{44zLTRJms1d8lq2fO+oG!VL#&(w@ras*k3%Fb{~BEO7Xm zRhos23a8+ON`FS=P)F+T*NuM^YUP8y^)z$~H)qX}8kwTxKCaX85zCib{PLjjS85-u zxOrDvhVmf)lR{;aZ^~8JfrGm51tjGFVceWLM=i1Wdc5lACyT3U7+&9m{`rQttSXtBB#7ainK^{?|)vRly3+g7q>`CK73P z!RLJ}hAdK&f6aA?Bg8c8=Xsy{B}r+?b*=%uqh%{QvvHo3X!5gt$r>Cxziy!MQ6>At z`zmGe#0*t~^ASw$I%j;p8ar5;Y@#^lBLXW8cFg zR^3iNHJlGNLO95rd)-3RQOk>UMi2KipYjZ#v2<_Kc&k||B(aQp!UaqHkbQ2QK3hax zZ+rTQHC`dd4f68Jh`Eg_x78t@hLwj^;FZii_Koj96L=XF6-_?O&Z$nF|g z6U8Pk^ABhCJb_V&7}zy^fYWT3ITRIS`)&X?G|*9^M!PUohyU56@pxrYZwFR21gZ*A z&oN0b-1z6Xo}Atil;l}IEreCDVn3U~N(cIuTr>RXd%89Jsx&Dv49&a1e)HFYOANvk zUu>;3+_7%qSV(ed15x|S!|KiW@O0lS^u2T{XETDc;{1Um7%0z&8-BMb_^M>zgzfN? zTW*R2NBI1;_UcubXkSK}!cH7Ta6`+EBlBkNDmhzH0s75!<5a#h`r%BUt_g-EXW{^? z_)_Qi!bqvIM0Xr)|3j(e1#F8u9luZ72#K)%(cGDk%J;u&ZTBb*HHyCmGR}S&PGXR5 zfQ0Z*E&N;-f99oexxPJdnqGjf{+%1K#`b4F+nX)vxm6AxRjF9%G@3zO{06MJ z`2Az|R(fK3ksa}6Eo@9QE6!p3>={$6ZTqNtG~uMjwm%xfW8BIoy3i!WRJN=4K<&_% zNPTTe$#Z*vV?aCl)S^`L7(=0#^gPOlx-kF zKaJ+@hOqK+ONw48Zd7lg-aXYFcrKjVjavee z^Z`>Dpj+(1++IyXO}Y)uvCCn^<8ekTYgNEuH{Uqi`Tn+sE&=9Q3UP^#LqN)#FMEhR zW!n3k&hoDkXmNh>XzB;J!pc1r!z(LE@fq?piWZW2=JFLc?E~Q9jsZbPxIJ27gYRwp z5k{TKSuRw3V%}Z1XJ4QCL<-#pTo+l9JHi~Du0^?cdhq()j7QZb+R)Zq8wnB3(1rm5 z5Q@hjP>(Iaeh;fbaDwo}K|{W1m8PT>>yzfwinRyUR!KgYtQxxumR%{vXGps&X2YKG zd%;L2Otx;*% z0yhqqCA3l4+7Dcawvw%v>>S2FeZ{{3`6s_|!*-61l@J?ka{}}hT8@Quh36i#X|sR7rEcvf=4H)GHNvQ+uq*v~sGWNI~YXx$06ONw1L_3{DcXD;zezv-Bf zSs6=vW8p|L7}L>p6U!!2PkD;7GZtC&9jg|kEa3M6&XJ{;*?P+}twJp4`THz(QxC&$ zZ8=qUTS6x@TricXBR^TjdO0Cc-aKX8)rz=O^6qMQBF5s-b$(&HrfA7)_n>_DXOm$C zweU(&)S-I$7lK3eC_F)oW$MAM*wXw%DXS?Xj3V$#yu_5QE40mL@Rj5@rmitn;pac=w#FQieNSXC`R;IDe_v zmYmsP@~gvZ!VZ*P#D&1%#L0JWw~+>}De(-ukGWkA&>!Oqz{D9d^n7j`Uh3d; z$A;w~J2;woVyVM|)&>{V9h~)^#ifrqJe~PbHBNcwf(h|r9OrtE_-)RI4}Y+|ZvQ1B z1n6FP50Jarf7E)l943PLf5uK9XHv;Zue-P5#DMKmQnu?y0RFp`3H%g?6qx8UqiJk6lle2i`m*iWgvINsLSk=@Ky#@MW%g#g1TdF!!Dz_skCqZKXR~;?_qvgktLxWq}t_w4DuM02!E!U8s2uRGf{;#Ov zH?y&YAGyuB1H(>RGjL0K^$&T^qs^0%^wr1dtiqKsW(jvWXRf>ZcCoBrJS=K;oko9G z%((+}cYkiUhop?u1YCEzA%N9qj;z0OO?}{~qfUJBq#$-;@*N!mlBct>*X`I}Bvo5~ zqI%;W;jYOgU;Q#U4)Hl(J*$tD%W_-a-6k30Z z`sFOgTi@)+492YJ^?T%5e18udmlDWj1@Z#?2Z=((JozF^nSn^d0r!@lbnjQ?z2iJ87A}NWU#wL%#=tI{!ys# zq`N8~$42qn`J?ZbXdRU%kYsoi7PW zQL#p&n*Vq~+7la}N#{312ayk-Ye6&=wV2MIn>zGus9|tuo_0rh6Z9^NH^?aZpf5Gb z`*4R5IpcDQ;zam3=nZu-00LJEg@X%ed7=3)Yn6lY&)@eHwHjl0;PcQs_Lfu6IcFRF zeG~SRa$>TTA;O<5lq-zsP2ES%{xsi%mUsK}U0m)3i3DAJ>wksl+5Z*aOSs4Thdb86 z>eMcw%wVQ5g3gC+xhOuh5*}TY%BVi`Mr0##6moL zQ`Uz8VQPc@-p{rMhh5b4<_Le0yY)k=B_76otpE+opv>1n6Q=VGg0R~h;$sO`-}4sU z!3|$xdh5CPMIFtCgoPk=#__4y9^UYzHl2yz85$T0oA$MFn>4o$Ew6=1UGB3gFxmDy zSf#fGB#s;{V5U9fZ}LJl8P;iUseja3CFK4eqvglEz16r-#I;XNjUaI$w;$Ou<+G!e zqW_ z7oN-SHCajmZptnRAGI?kmc2z8YBzsy^?Jkn?8>#5LC`W=q-Rv8a^(x3tJaDVsGg2` zDu$r{JyYbuy30;GI95*{!}9@K>%bd(FF=(UoN?+#&$~WC7C(VT$R+UF1ucIVaKio% z$r$beX*`?qSH(6FC`KudccNMa__7>tU(&2t%(;uj<&kWcTKjddjm}z3QY{ ztFGx&FClGBYKHYrYYfP@>3l*+7=l(#w0IOFd*SHd4!5H0A-87!(Pak(KR-k2B;xDC z=;6xfGL(tgO<7D&4m|9ENk3GcZ}sBmHV6iNOztbr6r6kH!d)%BwHd`EwNozVbl;Ji zbdTx6nEpe$(nA1M95gzgMuRLA_-6LNyssqYUWbn1=w#}V`z@PbdW!$nQ;xM`(hDr{ zFxsGlF5TB~PsOGvOEy$+o(^V^3#SxgCoDKpXa!@K0HU$TvOYRuiwGOO60|ryDOLhx zbUeYu@wi@>8})%WwTtK z``?k`a1Fslc!Yzvw0_i#5Q@6)ior$4h|{z7McT7UeEV? zhhzY|0${~vydlpPYHokeSO0Fk!`}Y|ZxgjNx$x4tjsY|O4a(`542la1D~D!y5SCTO z~s`h=pMPEXVUW#tqVPzpn`fBa8>SS-(x9OMXkKIRcVXGh>~chp_bU#% zkhyD#BB4=9G>P+erss9!5(lC^u<;{y?ay~w)?E%43dY*9Uvq6XId--iC5C4ddwapL z@v4ZH{qSp$MX@`X3JjA$z&p?CSHq5x-4h|-K+2Xg*<5gXalW6h{Bq?tCG5pL-z3A^ zom-NY*C_&!(x$HF^4oTpX|xK0@Wn23VvcdFy0MV)(H{w zY3A|KbL;U4Dr@<)6#br*=g~bQYv3dRNuH*eh0~D^=Sf+ad&7_VpSzUe4wmBCq*u|y zG>`CFuiFYeQZBM6&!8`wuCt=NKkj#JZ`^tC(hXmPy)xF%{CdN|NjXh{@=JYzD(yxC z6m%W23=Nn^-pp5`SQx>T?-XzxGVAh8iFAgEJuKTtw25P{!X%Z|uSs`&X0u~SugWi- zZOAo0rx6hO!o$EQ^l)FRvRZ3mN|%jpWMwO&?f#lw0-H;k?sTdQ*04QHtfPQU)?~lRWW&J=|VyNKTJrUc4ObsT7c_31GQU=WmMZ3dsXt$l5tq zeL3;Z9AQ1|K1jTgH-WOkikyBUNo-UL#Up({7a^3z>-vfG1YJT|ig;EE_di)-^J`rM z1iqo%;WMeACILvrdE>{e4A^|VFJHwf0-#7YQ#d-jppdR+Mi7-Qk+AoeYr%DMKoY%0 za*Y{sXi zt;D$gtgRZ0w?tQG8%wL4lc^C4xr?`N3^kSwYjb-7qwC=Q$lIdeJBxC1r-VqnzZ9@I zvD92MWu69O>QUGL5x!8VQ|88o)QXgyfE|AuEwy7c#rfTRx1@)MEgGrw-r_{-ueD|1 zar1MX+*^#iIFG(&I(+iP<(9eY;=cS3Gwk+nX4s=^qHa`9*5)ZcMpxNviTAItuMKw{ zQ$SsNb=dd3TxsiEKTc<@lG-G4Dt*BG-4+Ia#rIKF7WcSu$tHW<+N5U)9rHhxDbVmw z9QyY3p1{Ds&=vYrQ6d56aiwy!&D;4tB@-+set8yr1rwG}Z+C8p1qW3ZOK%jLU7KJB z6;smJw&nMN`j3!9A^bu%AEn*2)Ou}H z{3^U9diaMiwp#n>h&T{4bT)4MC%plCQjx*WO8KVQ@L`IA`P%iRtU(0wcXn?>rd&mD zc;9p?HMx`ReM9nySJJ&xKk&P+z+A$4E*%V!-q|;hhVr2zmL9shW_`TS9fDf|*VzAg zVt_$jc0Jq-&2c=C&{jw(AzNZP8KUIkju2yY;xm26D-8t_=6Hl&`GoH#_eRtJz)twH z>lvs9kDsI>{ibr&AbPl&Vi@4c8ht)BEl{svntN+UWz9G9LFx}nELS#^B^G<89c0y- zDHzV61YuTCC4*#Q%sKlM3QU)Gu~;__A89Z+>bVdpJ$@ilG+ zX*0zjsfCdRXQO+#?Cs}Phdh=&JsA~DZm*KRpBD|J1N6O6!mi zw{xt?D66g#-%|zVQfUb(lta3U!_*i3ykK>zHeh1&hv)r~qbuS~ty$nci5_5I20-_v zOI87!7)f?HNkg-=rVIJI4z}ft(|RuvZ2O!hD&zUE&Ro2ZcD)vZppINLO=^&W0P@D@ ziNon_OWIPn0fM4dn=;F5^WtJ2%%R;-7s8K0>A8~MdHXq3mgt3JmzvZu9i1M(bnCX% zJ23NGh_fCviuxzo+FwO_37Dn^@ea=3GMir|Qv~@uKE>@i`w+})Sot~F$3euxit6@8 zIlCMo$8WOj-938gKIi z{f5i^{q7v%M@+}19^1IN3LghrMJz0l3ct+Ef?yP7c?FRtB`zi+v`BPF#hC^o?S48U zl{vxLZyr2>dGf1WG=69#%IU-V+x@bAAxx@iX{Rc_Cfby@Z5ivX8OO2GdR|Gc$04Fn02K_c>**+K=na=Xc&Ee%f($+n|tPIk|B=MiR3{j#lSS ze8VF{RF9=*vJvv-OiL63FZCz@77nr$1q&yogGEwUcmW?ws?}zcj>CK5+$gzW(>ltKQ?y{*x-+LCY#dw z;W^h*jEH!JixudUIWz3ta$2vxjwo@?Pp-ML5ez-m*Y%S6ibZmLYCHLDZJYj!uSi4^ z%iyuoI*&=$vw!rpIAieI`4^AMdtC90UM&Er{6U^Z7(#MKBkc3z1x#!*8NG104m<*! z6E<_CXN>dp^TF1Am%Nk=-{PO}&ve=~c#-N~$5|f|zdB)VEOr`~Y#J=i9#Zsa__q1Y zIV}~zF>$_`-3CFbSij#RQBx-1WcjL3!z3;*>Gm1od{cOs9tV?9?g~=$OW)C74T5Z` z@u(VXm!RPuq0UgNwL)UmJ>GsBeXL!tSOs+kb8vVkvWZ2vi_w4<;TCev}|yKODS z*`GZ=)H`~<`DW!iCxmKGScWWFZeKw!jn_m~5VZP{Oog{C$13K879MFHe$8)04psZJ zdQ_VA5aYY}aU&Z--c6hDjLhSoEkr*akxp5+%r<6UUIaNER=o7rc~j(=vFMpdFm+qT ztHvHRyHr+~^8O4abcm68#Feq5>Yf0@Wf=81H7{Qc>xhcn2R7GV6b2S@C!b=&bTUy@kWr+)R{9D(CP6l{9Ntpih#9j$&$~X+t|&EVIHX}I4Orc7-7F`Yn|NC z-^_?3it0Dh+rfzAE~q$JWGrGhEkIpBV#gb|xacrvUvfXdV&rgET|7XKN}B|LsQc3` z)9SV4j=!L_nsamC5)p0-X~`s2pIUy8W4IHM&$-w3(=g#|{H9yPg?6g(?ms)J0TY{d ziIx%T{5bk(_w?oWO{wbBb<9%bAwBJe;)1z8%6XYED(7Pt$MQtA+z&RV4PH^3M2{w5 zyC`3WNuElQM%BB2LE#D-#&OlYFidbG(h_-?_>`!Y;B&$EN$+=SE#^B+@%+@= zzb-rKX_8>OTxeSy!rO!oYJPlw5zvY;*vw_4Ml}2tjs7AJ3s9zX+dp!8vxHJKWcDsB z;6El+8piJ3i2dgR&3S8Dm2$kC%)8DClcGKA9TqOT7W`st&5hRmdfo->wMC2;H&?>E zOPPeK{EfRLInv z+nn^AA@GJs@eZpE5^`r>`E7hHkx~BXf~Wh!vc1HLHxoJz&<&vQ!eMi?`&<+PsT{8~ z>}{Wt0m72-mQi@#b)E27g+#d$DY|4ZnQu?9w%CG}6IDn|Y;E@bRjY-Ycbeur|CwK< zWdPL{uSfC(_EGVbuSy*I#+T2&PpN_$iwoz*kegEm^>?(go@RJ|8Wv<@RnTL4E;@oi zkqcol_9bv{nb*-j_OG5a`y%xE;eefO9&vF~D;lGzT>^iOJ>AFhg;j-Hi}@c0v$(eM z{R?|V2t@W8&J2#O-uA7N@;|_yJ6z_SyMOAQy!p%ZEW??T-3Y{moWD3{`?av+dDc^lNcLk&g80#L_ zyY;?YZT9O!t+=p|;L@|*S;U_CBsPp-9~R(xH8hgMkX&G$*Sypj+;9JZx2>|oXZh`0i@{z2$1lHJDT5X0J`~R?s?7_k zllO*Cs*M?&PV8u09X1W0&dZ274v(7@SZnkj?V0l)XB8$SVPCG6ct}{?ly^7%(mb!D z?f&@=WgebNKw$2AVg9~!;*}1OiwLpYodTPQNB(QQe&wh!3%a?V6@mltWjXHQ!$Qas zi0n2SNkwn>6m38DmwuDwD;z#>KIyJWN*35g^t_y>Ic>YbQjtAwGp;w+IZRNU{eb_1 zLD;UoIXjH?=6oCEH(r^0O*HbBA1fsrUVcYn5^hEv82ipf|J9q5D6y$3*=Z~1*S<%V z8Jmg41e)Gfi^8qrDXq$bYNv3r_0W1%#CZH5ad4`>IBis4DH{p4@Bi3OoMoM{!%{A+ zUsYX=)?zdNqeNrO$x<#iR~d%YuqV7d#ik_s0mNWfUF0mP1|$3SaR`hwM3D{Sg0?Gs);S_ZE(WX`qYBT zl(DHL5_**8sh76S7Gk2{<_JegBtmdoKs6e?=W&R8%7<(Q#!8nn>=Tg ze!z-Pw{6GZTxPUg_h}iW9V{0>4!>z|i8!b3?ASZx5oU3|_l~}m38{b_hR)kpP6?_e zrkI4;s~Gp3P}x#ILQiz={Hat{Wlm@xI@Fk*kI;73qPoYn z-&gp+g2T$e#?f3==INc@>h6-=_JLdJ564CI+p5|Mh8r;h6`vYc$vSzIy3s7UTrIwLvltOF*>BS8L^wjOJ8eqp@^BR^L(eyh}d zjGoIz=7v7~Q<*51)%rGyMI|_LhsRg1a-h17$FaGL1`EspP z`!=ZBkBY=3DB;zua83})gimYBc>coeH;k5I$}JbSj(dMhUsk}M_|wX5f6Kh} zDP`G7&(=BSHo^-QubYeZ2KU?3$=S%kkPjaKtyPWb2-yM5KpNlVxdG6;ES^^GO~JZ`MMD99>@tW*QNPA% z>da-N%Cc%aYPl&2u}93w{(=zm9x6WP!0#QU4Pzj1bzVIzxb4mY@tJbjsqBR{EMuAb zX{{|+^H=5!a`ig{eQnLAtBls@Ma*Zj4(H6jra6VX-O4v#5@V6%jf)I6-oq_S8)zHJ zO}w{PXQC-mE!BkY`AO_`(cz>GHm)|z(TF55ydY$`69VZ9JnY~4N}WM~XHPAcYRbn* ztqHDm>#>!P#?4PDu*ZRqx`vp1tjPVwi1vpsl_rS|DO5;2mw52(=dpj`Z zaKeybA*qij^CG&|ViKW8YgrzP&v~zgFcFiMR?TN;s!|N_Ga-4P_xLKnKt{N{?{3=F zwOZU9cjsGXL8tzV&JgBkTul$Yhsuys%9>G&y>REdh`bKnl{YGX|c?YwlnEx!jnUF&~}_hdXS1Cq3}hb{tn}#f66ir&f~c6B?E`AHAHlav$h@YdaJm6CbB!^;|B)ewZWA9nwpySAcNElJP3ND?Q#!%&jhQ{w~Z*F zy9&Y}Opf1MUegE9b*nHR)=*`}6Kt5I!yweNjz!G}t1 zpjZcCF-VOax$gi#1~0ZJN{?H(A;O1vde5-wRUe0!zDGo>BlO1P`Q5O#4<7wWPiUie zGw=P)aeQ1SQws$Bk)cGIw>F2<`*a@rPrG=j1=661fE#6)Ha`{P!YAA_?(R6_{z8`o zl4;6mGhebC$;Zh2sSdEUn+@F za=T5egosLD%6CR(r}!CE)s%bR$^;R6xh#sA&o}c>f*-~xc%?dwO8Zv+;GT0~CA|3Y z?DwW;5)S7Fo%F5jmnzq7;pSk3V2#w6jVZT!T(+y56TIO~a`HT+UIonRN}pamGA~MI zH_PU*0ARf|kjyvUx2I~~2cp2*m~Z_ktXsBX^@oLf$B2!DPChZ09f6mBxV9h!;XdBX z&p;Ub#CBG~JXB-OHBaUC@&$}R*Twe|%HYH8RYYQsc6KSq!%jTn-bg}x2dV(O)o*5_ zxzcsZmQ}wSR6#;tJ_!dSb$;`l&F<(c^|f&o3}ol&@_L%{{8M(PXK|0hF-RR!WfMLm z!*=ye>Uk5^klExK_C{*9xYh2zcmP4ws5A&*Y`s{@JRGHojM$8FTDD*XX6ide&^Ecm zqoadPG{guCqNrZyBwGQb`Dvd{w_c?Pc7@uc3oDHgXXKgvAfmy_z_!2%? zh}iQaX1}Ikf~CyubjxieMzY3sS>ibUf$*mN!J^0lo>|GcMz0h(s48!&vstj_E2C0+ zsRLv1cCt*zZ6hnlqttr?(c)E*$H|DK=XE_kq9crtvT=A&&y$vjv1v}L8TN&braSv) z;UQmLqKn`CRutW^UBQ*G9+#_$>-fh%t?PLmHZpN_T||&^p3AGMCS1xvgtx$v!D{T_ zadX)F8HgTqPkAga0rgrMf=N9SnfLXhqK3~!^}Ao9I=0=|t;e`QUf=J26sx^1)v8x* za@zG6>zJ*#dHs-78sCtBt%uD}yI3hda<{>xYwBUcTC#DXwVj8$*HGRUM~)GmVV8N>H@ zCXbu1wu;}Qegs|2W9j)tCK2&0&uX@AIx^`ih|kJ6?a!AowcU(PIsl1sEz+p|v`Y+6 z>Yw}4e7q>{cvxtmUMthKqF8xmoKhRA3SWu=s=YuI{FAd;o9i=2F4Qx%Ru9+N$~bgU zMbe&hbQFBfA1Z1(tMm`T4Gm@@A_@U(iKxL2rV#=6I5+cRiAMFq5RDteGQb7uk4 zk#9d6?LC(@tJ2#9MxiHUJYcV2cR$}7*YmzzG6yB}a6h`xYs!$^>UAu2@|0KUUP zDF6}kTj7Z!;OV)D^*HRtRVmjM5CC_jx_)%g9dKcVNU@6rK0Yi(d}j8eAO^S<1}z|6 zAfJu&$(qppWt90u$>uc~TWEBmo>yZ#S(h}70TwY+^MPOa1#<#8Hp`s$ewsBsC+!l% zB7yjb8~jG!%_xw7$4b3zAX4yZqk>HoiB9;2P0)O@jL^Y*;Ep7B z155;j0%Lmz&HE!fY!@zguR<^jGQ|nk0p zDqJ!3PpfXfyW=nQw^dGPtNNk5$o|vB{^B4_28;v!>)%!ppsl`AF#gkOAZQicre`PW zTRi=2*)}%Ee)DmpYdItvdYa<(Zazmh_D08;gZ6r97hJn)b%yEXBKh{XK_3((pnX_= zi}MgVU0%?IJXHQMO>AanR+p#+9o}bXN5s;-uFPhcIJ|XTe!l{D1pR?jAI(=od4HXgWR_F}1MA|%bdfR45+ ztdh~BKHa2?M$qf-(qzGI!Ry*;@yL4a^CjbKot2jFi@sT#wp}e$;T32nl)enY>YpK! zR)XUULZZ84?5!S#Ce#Gm#iae@3+Soa?;q{_nUyJMoCqB)tM{%ztz5Ttsk1y2wriM1 z6zc0#A9~me3zH3%ByK8)KSS5~J@mzlTYau&@(*#~x4h*38YIFFP|eb-{$~NFeowVU z&*z`Dle$`_hriUo1|HV)fAx{MqZv-vR>9}vC9vv>Bk;M*SHHz;wtDc8vh=t6KJe6Y z1KYW>B|X~I2pEK|wu5-fyFD$+P%$H!u(PupxVQVn4IiwhK#VNxl~)CsuD8ip8%WA| zLN2k9Y*9Ayh&na%-LaeCX9L(4J3OY?r<9#vY9L{^%3*4q$f}g-u2W&*D zSXemWm6s|oRn~L7D1^MqpgX8D+$7*0n*eNJ^Qj8`tMj&dD?Z&%qQ4&iqoUEs%Z9Kx zSv2m}%_;-SFcSr05x)J5)3P8=j#qjxnM3J#Z4g|a%?!g0dt8jeEuKDoTDP1Y1FWe{ zAWGkpIV>!Dhu;Emhmec_Nu#9CLx#G@fWJqfNWIeFJ5pN=ujAGRFkeQ2?YKcNHI@mCY2THAW}SBbyP zH5Sz1nd}!~nTJPxD}T7J>D8^SD&IvC8Xm6?gWK^C>i*CB=?KBK-Ab671W}(LAUd>Wwf|Sj`sDPONyQd01H((`C370#CJ`#AIjA*ubwd}*gLPCvC zx5iu9j4peB91rmgJF7A@9dgC|IahwaE(G@N#`i2=-&~K&qt-_6Agkb#Zfh1;q(KiR&r$3ZDnGhi8o;{WE&$z6C=P&|Tq2aD~1>_QA}`j6-H-!q6q0<2!7$3#6RmJs0`kMlWaUzU0x4|U6SKu!N! zXg|RBwR&DV02O+TfLcF07|HC@47g!ul}n^ z^zC-r)p)flUGTTM@GsLsXG5txWB=JAaEZY_kW6IM@n8aaR*R6&>2Y|=*eBV)UhjV| zER!%u>D&7KR^Y}_db0!gy&#zz&@K7`NdG(Nj`it-YK$=OX%QIK5J(JE>TtlPc2HUZ zfsjE(fp);Gc~4V;3UMX9nh(pYu=BIZ}M#Py6wLD={3@f%{O3SSWH2T zp9ij}|0OpD^d?E^mb;HUbVsCmu$Kq;m^7{zwo8r}x9#l1wRkgs#OOsqDlP!E6O;k>P;fu*BK=@{4RxlGPedb+u(7rC$aS}I7-zwg4@ z)vP8ue{enSHg{4saus!=l&05GIZe^a-FuKenfrpTMP{}?`8L&UKKh)%?W95Eo z3JkDGcw|m4o_6$NOY@x($u86@8a5FSYMc7wknrH2jQemFCG8=7_H1YvEiP8c2+=e- zc1R@*GK6N%bXJ9PJtaI{E86F%a8zk5O$_RaS5;wf^DgtajZl*oJR^1IxDOcO{bUKV zpF^+1F^e)5wSV^giS@Q45y$Y&U^DUm?9B!?iStGtsg*LVsn~2vChSTk4!^DHoe@%4 zQ`Jm_X0i>HRHe+r*K9ftUJQ8}Ovt$BPqr?4Gy~4iDKDpb9w&3`YnOhOdr6TfDM{#^ zii+k|2J#3)|EUov=``Vkn+#;fpW9T6SknsIaW$9q7d40_*wVHfv6$QHy6d=og*Ugi z(@~M?_ZHfu=Fe=@Xl;5#hV_vB?OU`CeWOp_#OF{*p#;GTov50&76Da3U5c4_T zs_STwpLm}1#AhYT4vz-xGL7=&|4?ytZUBBc3XT6RN-v5bl|8F-=;dw>N*p$>fkUSD ziGuTh)2UJO{>vpDzVfpMmDN;9cKasBNU_Me+Y2Afzz?)#;b(4+DUHFj*%)6`P%=H2A%MuL~Bs=^_G`nd1FrNe&+cvB0(k+HdQNKi6wUGd(vKj&PoxOec z!b)lfDnLOG?Hlkx{x!kA3x>+_2_j|5HB%7wm z8pKyCk`FQ9vm*alCaA+>1l$YIy|GAW6cp%%wLuEgfyx>Fnf|Kja8LKuWz(Z)fQA9Z zbCJ=pEjM5*p_<3(1v_$d4eu5Y<$$ztHLmGFA!#{7OXXe~b{;H6cJB~0y+&E0Z#kju zMAnKcv5!gXm{{-y(Y7b3M`Eym8LiM4OSAh%`zQ_ErnZ8~BM-VS?tzs+4rTy@3Qh1L z^AZIa;0OxKXJ@PW^q?#0)=rV zcYRCb=wKo~tz#ly+S~ytrCYxIqD*J6^^`<9@(obG{5AA_CM*O(lJt9JOCg7;B|&Qe zsbv((d#HmRM;L_IZmD6>4L_gKX#o!b-pF%+0)MCXvIr8gQdxbXl(GE5y5+9Ifn)Q& z`aJpItd00en^rg$8+vv?VxPi4e5rhb6utKCrZf|zep$?a7Onl6Fnh6lRG`>`={P|- zxq0SGesY*P?N;KrajWEm2j4gRAlV;_#EkcI^iz^qz3ukL(O4qjsDSvqm4NZXz^;c~ z0cmAp%qx(Ly-?oGK!q#gX*u{gR+GZ?KIeNZ5{ZzSqa;q!ruGKqvoDD?v=`4?a5R$b zy~m;fo$#M6_;bX3c?!goLWW*e%9clfnnfq$h}- zn)J!q)x8?aT*2A(e8!w zce@}^7g_>679r6CP)S)*@p;RHSE3}4uo?kU9zTY;TU==EZ_iCiZpH91DaFy5kv0L2 ziq!Odp7h?x~K}hA3~&nexgyZ<$Gv; z#*f0Aml0LGcTl7#(xnqnks1k7LkA0Dp)0)#QbUnmLKCF-5<(3~hXA4XgtLP8^ZegA zGiT;}IP=b&nfH@!cGlj%z1G@m|H^e;OM`Ry3-YbHw8+*##lI^t1kxHve)<+S9H6mE zrsJ@Z1tz~(R`RxDZ4X{Z zkN5GOy%^U&NrYo*!%4Y||4g;B9 zj+^tp|M5S!Nj~ekSj1w>(|NQT#TeDs#-7#0*QGt`YUeU&6ZAAShv(Q3VZtV|Dg!<2 z@KkUldH5$2l1%fa1QOd9;nEkMpe=rfKSHh|q-K~+TIY1@JMcOEJBadOD~INRaF`)9 z(aWm;pG*1@S3QU6ylX#g+yDy-_z`Jcf!s+PgC1-aU06aur4qDjW82az$dCDhhS}g6 zx6pPTk^HqQ7lL96)L;4BmLvHlkefo_AFZ$=-;B~%2C2HpL2JTX!U+4^@4G6d|LypH zgO!t+{%@r6_jZ57l)n%0H%9r}eSagPzlQ%C7M(0n|Hedr@A@|&I+^DGKf6J)wIl6n zU%E(pu^Vu3hYR(G=VlwjO02#N@)~E;!^B{SNcZPq7BKuA+NktpjGp}n5PP0$ywEaU zrif%X{@#e&jgERBRpObd#|O=hh}+2=5+H;4flYt@27at9*dK=Zv^{7mwhUV)ctwY= zP(Z~Pw{hyHE}X0Zo^R|L=eWXZusgHV&$toM?CT-zRY?sdj#0UJO?H(_M2wZK&iCBM zd9J)}W-(%={r{A-wc1rAZ9U$0@yL)8_S$&7A{15nRpd}&yXt!Y$r(mem)7ae%)E+C z&p?$7gjqnbm<7GgxEk@|*B|5UHihAuh!UBaO2IMy&FVGToj)!vBlHN^-g~)XAu7dl z5QzGJET@V@%}UWo;#E`)pR?;LopSMUcutn4kXPy@rn0i+wn23nRo zb$&hIb505$TR9+N=dzy}cIl*rbyemd;Z?J@U2tA+Vlh(E3qIWO9w9X&;m1sd)S=A! zQ>q}Or!~R3YBoYoKjeA+C`VB$L`yA zrcq^V4U{WhYsRz;r!a+>Wg6K8u;K3_T%emo(e~+C+s(bz>?lhHy45zxl^w*k)xvX0 z$5NXv)p4}2=F=ZpJ7~PEcgv-2;CF3&)jv=0s!Z|MwPwlRmPz2vG=Dv z*00K4nbVcLtFtSzj`s*puf+HyuYKilEI#PudD_lvZ?YJK-p!cw6>z8cxL?Ix)>pUa zTBLstCA>J8O5Tn1+B_67k8;tYL9iGEHz zROv6+eS*2a8GE#2T`#BJbB~5O^H)J98qd-m?sr1H8s*V46U>o3d-UoqN+DE|FeIU5G}&&v;rcGC#}L0FlD`ImkVMOv1HR?ULGKQT!Ks_Bz7FB30VRUxIvGjliQDO#XSN?WL<=p+J<03Ab zj9tW?qIsDgU!~_Z`TEtnjf3Aa|0Vv^i*ieCHQgfiPgl-7?$+Gjn5EV*3>ljvWS|$b zTK&ruA&|?e+6){;mYtGY6f0_=4CH;1V6=*NP;|8p+Puff(sw5IW$?Eee;%8=+1a|(s)PSQ4crlbX27AnGsR48e1o)50MYNX+c&9P^5K?7AqqqCniXdGTGev zd8}7Fj{%%+=r4f&Tl~9`sx_YK{fF#OUo9szG8)dnGFU#7HKB&j-!A*-a9bHWZ@_!5Yag;$X4ZlUk+1qHu}O~tM#q; zYGzgkaB+UCfF4l_mfld=q~7WIEDqst9n3dlzv2`kF?VIuL8A8o+1vMFC2FcZd-6B% z_`K3)y>rW}&*L3t7e;IEF-j)4%D^bk|1g@XkNSeR+mj5aH(7_Ltwh>Z5mJ=rO_%cA zi*h`>#3v{$qney~ts=$Y=t_6(p5d1e$Q>JyKf6rfl6fWBW6tA&*n7Y~o>u!MgxgCTPJeu#MloPnz8z8r09Z#ZYyS=Q4cE72&C z>Yp)mo|e`FSIe@jx)UpNG3^J92C}4lT>^q`U@oRsLOgf*zFO`4P~9)p_ISc8&DK9v zkk-CHEKS5}ZIY9+ZSL2LoMmV6>=4KWm64M&#$43l^QHcbkiWJ8)D&%CMMD5>*}t|$ zEt`QPIY+Ed{i%br&}#LZb`)$^SRmp&u}77LJmZ&)A#Ep5dF7DMx)ORPNv83z+{rt{ zQ&VnB^0ua>o6&i6N%Icd2}VFyz`6_o69b~dr7_37%5GEqi7 z>z9j|H=HqzA;xI?XoD)KF9hZr8BkD9zW|K zU}}jLe=7@gVp0>m5%jcI5e*Rzk%>ln@(PnKYll+06TFgvQ`aZ!KrGo~_VCBzLZX>1 zX?>PJZaW@w&+!Y{NG|QzlcVBTqX$)+_r829PIca#2iei(8Qal4c(xrsp1c2nf)6eH z$!9>EZEmj2`*7G}j%r&B{9c=tEPDqrZs3w^NTTNOO&IgI}15)(_R3X)+*MBRR zMPt1V7Bl{qTL4y4?Cu~lC08-a*GoVv;bYubkO!gf@mzsG-&54IHGv7v+_38Iaq#vA zd7~osp8PopBgquINds1X4TbT&b)z__!H(^(HEP{)x&He8hXgC*ov2pB6sxW*I`QP} z45EmKQvuI=?aD`Ff*@p#nE-~ z3kqa!-A07A;UYWHR%Q+rx>*jCP$m zW>?P3t{m}>S?`?mr2(sQ?EZ^b%dW&U_8KpZ)C03zo$T~knIt3qgNFd^A+v_jD!aS8Pp#tB?PnTDbzJ{#FzRMPpj=jS=fL+EF_z z+shSzJu)n;c*;nhCLW!hT+Wl2EhT4cb!ZAQ&}yJccg20}7v(9lWB_i(V5>d7PwIxC z^oMj^8F_PWhXh3UTLk=w8e54KI?yZ0(-HM@|HGXc%U8LWmBm6mHrYKcsgnw~{vJy- zymjB9RcGZPRKhJqk6|w(0hzElp`Gu#t&5d2zj>3t`-0?b_7gRe3&dc6#o1eirf9IO z{%DHWu#7n^<7#wE#MU>ziBL0I1tP|Rh%Jg1U*aaa+4`Sj!D>+Q{#(>s9Y;|%4FdG0 z&3sov>GbOqP-(h#8y3f)aj23nEA^0e8;`{9ZuM5vzFf+R!eYc~?ev~om6eRt?pUe0 zt=L##Cp5ed#`Euo1oaI)smB?i9TtFv1Q1)Gz<%)f<*|VW;LPm@ zQ@r?3$r`lPd`5w7@n<}aQ=5;)30Vc( zSBZUoj@vURePu{OVAZVm!%fsm|IbiOA={QE~aJo`jajeTh_H%=*-by2Eu8fZ^@ zquQ>ZdS4Cwd_(sjT(n8tGSI52d}kYDb{HJsyi$V6Hm&w3o&G~Ay&4a~S=9#ry)X&& z-jpBH#PqzLo|LyXGH1Q9*e+1}B*u{a-``|Ez05J#5htbr5H`9Ak}j{r&H!W^+G#*b zd9mhjwf`*z^{eLAr++g_IRMB#6|xSS2MA>nzhN6cVHO2-jlt(ocE*We9&siB(&g1%*~{@J#Up?0f6)<#|q^yIi0`3r-pQ zMGUw8pIbrxPX~eu#{ZUSf%F0^5rFgrgPO^sC4*W39wp7vlUWaNRVNb@0KUM<{lD{4 z-*0AS#u`Aj095RUdV(Ygy$|r4Cvee&*3Qmt??v!IN#5F^=YUK#Pp>chY&fq2;47V~ z;JV;TqeMpj8x$5u|NqSnN;Fm{uMdP&s*#V^@+1A7R4jrxfFF_$NbL6i)8PMAFBv*h zhnm=#iRwGebvT5YZMBb4960Su+D#HJ?pXOrm4P&>A zzuB3ZUc9(iLF(-^uy)c}_43K@z{cpAY7U*)qD(BZv$W#!n8kM(?H>8_t;N%OEOjX0 zanBcN(4I<>?*7;GaY?WY!8x1&2z*BjUJx{{%P@j#B|C$~a4KhpnoiY(4J{$3Ghwg3 zk-PRDgqJ0YxC}H(UtoL5rEJyPJTa$xij=4TZAdIx2ZlTBRi8{=lZETyrN^g6A!V&l z!~(mR88Uc2F*%lrzkTyHvM(mY21~Zd|5fY3WsYGZ;bMR3nr4T^HQR9-BAzk34K5n zy7yYo5BwBzhlwrc27Ad6B@l$?`u6%Oyi#zE9TkHTP-_jo8WFDVquUJh03uNme3fd ztj+M3O3~>f4KBzlkpGVHHSblK0m>z%HI5FGe(?bjHW~96V|dlEj0?Z7_4ME!$(o_+ zC+7LE%s_drXJ&pgui{ezM$iS{C|!Zaj>@K7I|~}4(4H<%$LpTAy=K$enG9P zKVlr-*C~@X=$|o2#b283+`|LgV5ht7+3Hl8oPAv4As50;gFfivuly5f>^xl(g&fav zd1txk#*9rDD3c<-BYA-@06OKP(6LfFSQ}~~tx!32eT=B;DZG*g8i_@_Zc4JNl1*)V z(;ark`=Z)#(O*0khj(jxyQvvr!`Yp4^`;Uqk)1Z9H}&<AmcpqIKR~ z1oB-4Cb>uvelzOO)U?!*tir<{!6rjn^@>?FzO%!2mfb14&WAP;e78P{k&<>|HTiii zM|gi#diBLDhc6Ej-Mbh%-Hslh9}BTJL!p?e+D=dWVX^zHBcE~UL!E;JR`D|`73Wj8 znhPzk?NKl^!|{`|l-(95+AEfkDsWxd71!!Rs|8eas-2Um_HMeng^tk*ay!rEOh{A- zWW5=>{bc*lt-zI%uTq&hvG!>RA-_{sDfa`PNdD$%?kBN9(Fs^tcA4jsqKyca;SuR+ zE~t?hp=WagtFhTZ;lHYVGU)(|15CQ0-WK{)Nb7J`quIHB5jF19qKVMO<}A!W^&I(a z(1!p)WJD96_gZcNAM1<^78FZ{&c@-0On*Y)VW@qb@z=$M;^oqkE0n$K_1o=Q!W~*O) zbA+w}w%w;l-?`g+cqHIET2=47#zH7c^o>*_rC1NxjAw0e0Z`q3-hAm!!IIE3q`C8t z-^tu*$_#6vVB}a`;#@pZG z2U1?M6jrcokK7^#%{yPkz+$On4-|qU<@#^`R1KQ{c0ZQA9B*RhnNgG-&C;{Fq;E48 zxpvF1;ApNuKQr66R7HzAROXV)bQtmCuhh91YU=oc&=YY5_R9R?R5%7f*~ycl<#(k! zt@OC!8)mqhce_T1qIHm6!RU1GVwfJD{a~ce?=&OFqOBu7C;9D7gJ3;5&*1%|>ZDei zkl3iWoj~2@bDK4N!DyNOQv%YpGMBbOE(L47J456AYSZT+-A@tth(JUP_y@~9O$*;2 z@|Q!n98Cg{k9Nz7Tj0ncBfoO)?EYxGIeO93px#;HwABx>LGPl5&hsr_i?;qL*Lnq;6n}hG|Pz&c@3u@fOXIcz%nvEFddGAUX?YzX*dAYqdl=VuYJKFT4tn zUHLOXX#VIxH*mYMe4e9vt=E|k+a^p4p5dk_$=5k=C(ZVF<}Ec{_ppY|dKVNhxnxlY zJertOZ$-a%nx4Z7wwJwTJS~MSzxkTsLKF{_X|^5d^_`0+Lb%vY;! zm78IBbF1f(<-U^vQ;DZfwY#DUS$OaLe9Eh{qP4&*Ji+eB1d$kzSiS-?v7vvu`)$(& zisuxbi&Gpi?eTd?8jOFE!Sw2=?7dR)g7q|-tC>P&t>DIIdW48e(lhv~y|3C;^5$4^ zB?M#PXw$XB?O*7*5AI=9p#)}MXhhu`xpst4cY5(_?mqIDwX&;560cb!l)QbzQ(Ny0 z*K!qA)I!!5;xE8DcpgxPsxz~A2*%L)({sq8d@+Z#u7g}6Z9@##YRbpB^~)kW&K(-T zzR0AW0aktFCv|TQ%jYoF_h1KKm{L5;P$nWmMq5oC_ML(6b9gDUIGR_|wJJ}QoI(XU zE($hhD^xa@ir~{D+t7-t5=wene((-#slZJ^6*_&;0ldzaolk3<=J*u?EEcBpH-pg^ z0aphk^cuZPFn=nKR@O55X<7`^IgJ_TuYfvhg_Yf=9}_cEkNe4Q@e zJ5G^qwEIt2Tl6?SyXY8Fyc%1%|D{SyE}cT~T4s)Y-gfCV6z9U~~xuvJDIq;Kz`dJ(-LHd))>`N;iP((2tmcSlcwV5XW z$~ZtQs$d+{T`7hSKUL~(vTK%DQ{FFOa}x%|6x{_jvHx2w<i<7j~101iJ$G&d&RJ}D0khx_4)Xw+O>-tl%EN+=9 z)S(L9vsZj${@7%F|3U?mKW{r%9TWk_Ce6$pdYs9g_Lg<=JgHCs)yFU;?UO#e~4KuvcwI?t2&Amn>kda9>S)=#PN znViA)6=Za`l$!E5F=+M)-!BT4{TG)xB=4$K2S1{gj!K0S#}2zbM4{}eDoHA45n5la z)p{l=CR%qvhWEQ7vKK}>ThjuI68j3@5?P%!CngLhcnbpSU*@?>0P{5+L7`tD7?WPf4Qhz-SL;)WcW3_p__9r;;uL1CLOECOWsJo zG0%7IJ&b!dv1Sb*87yX)G-Cgy476TzukVPQK>qk<;vj``w9s4eGo(bv#nXRBDj7$!x2l6)7& z-0Gma?!{%=l7XeBC)Ee7m{?+e$iYD>qsy@_XZGh&FWEd{^vX_{D8;x&UTKwMm>_w+ zoLa<^7{+iuF{4XkKjh7ynTn2nO}x`Odo+4BFDE_Wzn69&$-|%q_k{ZiuHf71@lI!a0E=q@O1f)i|x4YX; z9-|~Wa@*#Edcy08&Y*+*z?MTKPvFBJ;0b`2BtKp)Rxr6FJC>!Fq6w;%f!&W8^sgRe zwTpKyi=}uRde=c;3weJ6XMXIU0BRd<`2@pDrk+MIySq9`KD)3U z^HgEiJ^>@#%%}O)fuygdz0?Q1?{{`$AVU}L8TsL4>pdy+G_ zmr6<&4gAdphRP3-pPK5JUf*jA0o9P$DDZS3)CwZG%O`mv$VUJ**MN>=(h~^ZGiZtax3RagcdawggDT^J_dJlQ{5jp2|3)cs}B7`RH6W5xai0*#N$ozK2eRDq+@w!&^`C)PRP{%%W>;} z&u@{=Ymbr0+@$k<3t)s^{17K`%}<`L0N}!ZYif~*#3vTKAX)G|rGrFiCN=m?`k0xk z|Ic}Vv-)_?oT0^``Kpz%$DY&FCr-hCo07WW9jIqDm+q}u9qp~pg`eEgfV8e8C;!)k zA<`tfIrd)@mw|~6FT_Zca?Q2$_QH6GNJQ>CP^GPpoz2gc7$h#>FpGh_V!UI|RoAWSmTsr;4dIvvY zkL@!BNu_H#P*+j4ku$xD)gR!748KW9t zB^^0#N4Q56r7m*8>5YyWnO7nF_dz)hT&No;bXoH=C9an}1NR8TuhjDz{H)t49lX3V zUhBCFA00I;deKPbw73@NTjW|+GIvbGA9tj!tc;d*@3;=~LMsPMB&!;~Fph%T9TV`r zbA5M$9$S#T@RINa;0O`|#i;aE7NA5^tVDt$SXSi5Y&$Mcs7n*nBDX=cbbXeVgWcS` zb!hFXq;n=oT>@}>hk`jNEJ#Xq)x$!f4_Dw;LF+^QieMtDzpydjr|J?V$HGWcPoAE|A_Ga{}iQTt{p`yXK+vI~_nJ3g&4H z`*4ow1sDtW`2;I)M8$J8(6M#Zb#M%zKtKbMViW1oK<{v_Zzq!1@P(%o431h61-%F_ z|8oY6K#?t)+sWpe5)ilA2Wyb`@oR3q9P7gnP%wS03h-1pQmf?(fL|=AtO7}k(Ry+@ zW9=*r#YgWu@<0d!2zk>T{w@Lnt2Ly$?cA{fHy1s{$9sy)DPQ;t36#%%gHRr2GUd+K>=sv%5?O-M5bvYKuV2BOoU4jnJTvWRIjQ@P=;0p(i4D?+dK<< zU{E`NBP?#T3m}SpUM}xw(wVnakFVq~fzok>vzvV_p%jb**Q=$P0teDG4IIAffA73C zoHT*h|4{Yk8jFpN9&Ap&qm5Plusa)NMw*IdKKqSea1v!a!BG;xKE$bh5CiI2r0KXE ztQn2M*NlP@5FfrHD!R1-hjq@he(g+aKphhg!8Fy_T^;Yd)c}gK6W}cg(i*xvN5V>g zv27}trz-T?cLdWx2tFnbx;W8&j)~O8R^4piO(Lo6+1Xol0_cY=7_u({HP&Fr0Gs~> zzg_nbMeB+y?#Ea4jTI5Uf$%(p=7DOnHvvFU0oW~%UI4MPJdDoAVB5Z-5X ztil6GIHD^7OxHX`3kXSMNW7l(bEk%+b%Hu|j(MpByJ1AXqwT@LH3 zizBdn7iU)Xn*l&9FYuinQ>};5v0IS$r04e1EKG_lSy<@&MOW}}NPrW#KRpBFx^m=> zkNQohc`LL-8xWZR?RX=100>uYk5suA zRh)*@J$s@rmj2GDSv8fHxQfDe+yX1xV@7kO9EYYRAk>%a@&o7%M5l&cr>X2yMdgbBffe! z7IU2`?J3y{*1c9EpDUD`FOGE~ubadnF%ESC$5(8>adey}3!fM%fw_PJ{r>qqc)ULa zWHR2tq}3Q+(O3qbaje(^PZEdWU`l%eF0-+3kY94rC}y`ig3 z;-+W)X0~dN2^yU7uOCcK6gwy}Lgag|psK-=sEn^U%ya)IT=y~H5Ak4t{y|+7${`ru zMhm&amU_CN>$C)5uS-as3;l7hi)Y^}11hB!d|;)kIs@_R+;hw_;_9I=s%ZkM`J)JRe8&WtPi{fby0^hyJP02RFIFi zaR#1%XAkbH@)RJSFm`3*bB7KsLo~O=#4zAqbktzc+Dza&^GSe&(F@6W(fIjI#0Jo# zXeXUj^F3Nu208(Hxv}Tjq^(U)afC5h{T3n;$RRJxu~qIh?4JWy`hXTubky~*TXtkX zIWXz8UQ4&^;j2w;h&(SzobT?O<4+qf9~a+HvYpLg-(Ku*uDbk*7u?P~it(6z6zCXO zkxcdHO!5-EJI8Fc1Ad&A;Gm>^e4qscH#$|rS^x~MfXFQ2OY}OqY)8C=(A_yf+l(|;S>5lk7}ps_to z4tq7g9?hkxJbh&72c%xb)|3aoD^Q-;lmPVf-V{Wci396N(yJ7MA*6+AQ~PE%Fvth! z{?)M?0!S6feJmIN0taMqMe_wMd= zCCZFeU6#LI0jxlm*_+AiO}?AE1@SZR0BQwZln+D%ofkAu?s%d&U`y)OG*GYyF3&&w ze0-g|_Ql}C3|c_yfSY{6s0rO8In-zWcN~&FHthhf(*a-s1;s#?#T%@EH8}`jN5b~Q~TfD#A znTuvdMu2!=4HRrhFouFHFjlreEu`iMgaD*5`qvP@-vF!tk$3ywaEJj@4GMI!mc}ZI z-2Z_<)=1&ciJ}994rq$V>X2~aOdz^(=ekIubN;6%Qk6r_=dfE?TE<3QC&f4avaQZ4 zk%g0_7XLr)T@^1N)j9~UW;xAlo*h+y&!PIi3`OW(Utgcsu@aAd-YZj}#hwOqkd1+~ z%o~n^6U#m!$O?fMx>*YmCn;mdyQTer3}?919;bjs=3%s%Fk;`e7Z(vaHf~gOchBj_ z0!y{>aqCp;_;>ov2dju*+?#C=zapz)mbiZX#GX?KY$kt{S<06#`wVLS3KNWTt=g^( zv9PJzAWP=fnsO}4+<%S`zRSjZY|4-W%F}Ug9m`<*`TNIgI)+tLp}@_>RpRpcJ@M66 zrsQ>8RG}BYp;VC!mo2>eQKk}jF+S50L(js_Lq9HkDZP$! zK0a9l`Awf*f?EHnGAGzeh;A% zptAFyT&@_Yn6w)o{a(PGK`Dpva!5lyp8n#pi*zdH3s!6)YE))@Za**?%N}Y&5m{V=6d!B%URRg^AtE57HhdlO`7`;C z(^sQ&5=+M4IP|&y<$PHodc6e*$t9E>hdk&V+*i2nOks`2>d2CW#7rlb|3o86fiJ9JlT;Q+yQQw-ByY zo>&pampuhMT{@P_9EyN3|A{+@q9soNEi@y-{uwcw>2V_gyQ&TCttgU&dIhV?REA66 zr+Ck#9~)dVOTBUzF9>8k@5skf>3i(~b)zdzk4VA3_FF*WcAIYNv5BExM*QNX2d~a= zrA0n@6MEQ#j3O7PSf4nkZQXXK|a&sr^qr%ulyUl@-KE|Z9`t7Vru7xyp`?pp_dV*30{b|`MsdIaYhGx}?Yr6aiV;Dg2cT<@b+PCVw{Jx@ zj6eHVN94RWB7b!5$g^9i7$f$j%=FJs$fuNeZ)0!#pS}oO317I_-B_G(1!W0r`uib3 z9lu(3y`a!A$K#_~C)hE43Pq@^&p)nas8 zVinogtX7&wMwWU0{1(B8lCpdRWnvaot{Ro(6PWF05-C6)HO8c?TkGyQ9xkC&Br3^5 zN1GFq-Al+DGz`kl3bZi)JJHJYa z@ygAB!;#b3Zf5zu3PXn;NLGwq*}0_jpf~t%f|2YZ7h$u|G8@`uS92SW<1$n3lpIHv z&B2Q%40w_u66H&(GJ}LyS7Fkn#~%8X+-zvDdBtI@r5*R=Xa@K={>Gfz#25p$LiQ~N|rd#<2vh%S=H z!)9;(i*64Ow+ES{JS;s}uNK*@ooZ{bL9`64^%vB2d4=Q|fbGpY9*M3((*=~b`sl4# zk=cWo$?DqTjgo=Aj;>0pJ*xwut;-%*`Uo@ADmAeyzTKEot?E$vo?>P`XNz40U!0iC zQP$wW1o1DP>DMs>QDUS@Fc5qkHX6^4_wVzHybt+XW(k|DooKNF zdYl7iU|^R{m-IDAP=_;WZo5U#MT_SpL<;kd+B=Ka+%U z-)Ycb?;tScT1mWay83Ju!;@4&$eh@*^|aENSA*XUjX>GJk_j;QgV5$<8%OUd?a2BR z^PhNtTc^>lRhn!J$eM+yPgowCGG;+O^&Kue?7B{4$D|F^E^r;8T3c6YooN6!ck;;ccW60+#)FwO%w3+Lnn3 z6RaIJsB1IMEA93&(q}cs_SQxSx{PKY{}zedeNijBq}}_=j`r*Pz%QeP+UFN`rMiai zD!vnte$3=w-K|uG-`2wJ9^hX4O5bBeP)!v>xNQHXN^h2^P=#K_I#ipqAReb7w||*# zdLZLz4s7LF(7emVdzL1PXmob>!(8f3;S^L2Q||WX1-4T!F+hQ%p|In5fJNqX9CC|L zH<)cyCz-WA{SeJiz3o19y^RMfABR5uNX*szhO0R|<$c_^_wZDa{5rEIbhIa4&`oVo zmg|X@jEhG`zgU+3Ov$FCVh!SaH9HMO^z_kch+?g|5T4vp$q++3q0H66Du1`D=#jH?;ZgFR8XC>3r zRFxx3`ip8u2lj%gL+{jt1i!+tM17-iHv=0bY@Mkl;&$qNI~|_wE?<*G)zQrLlsoUW zdT*6m$r57-Or{M#P!RRbP_;b^9pbMx zONE>Gd(zn>YtALO9@CXB>mS*!8G_sV{V!VB=MrKZ#cWLGnXkReQe7=98QALBME?o@ zg5D`Ub6;os=n)d5V|GU57bx^(WdwJY<)~T&Tj%J_m+2q6@(mdHBDM$7^4?d}^`MNNV zT^xl`ZT`Y;BFa15gDvjZ+c84oL6C7mKG@YLOYvRXe(lFgqsqqu8HD)CykCC3$O#!kCdHF8Q4x&jVyF^=I8CAy8T1_j_ ztUSSetO9=Cbwgq9*KDEiW>#M17|LN{F}&aTWlLKqYUGpTM0rxN-bhjkoD4Wj#Gd*kyL&m1$ncO%cGx zuMW#&A*#1Xrp1&^3@l`@59Bh6c8Ph`Md)dWc~y!`R4S}~z_=LJ^i8B~wKZopy05q_ z-eSz`Y<1DS#&_%%k5A6lf=&RDCB+W0Y=gy4BLr`KX1d(hPTukli4Cl`oHsg50^^^dL9{A|@(fm*%ce zJr{jwG8?;)Uf_F1?ZVjR=$ECZV$P?qX@x6hB2nTu5i6UvjDj?RxkMM{vUg02qXgk3+)o~O<08O5-s3C9V96^}$&&s0?v6hRRX*p$>KPjs=jq?jxd*p7DO8Fllmi^~sxzs1=FQg62DhUe zwbYsu9J1icyYEd8UqwdZd6%QMcsp5~ZB_0~YVUvbSRza^{@Bkq&2O?$(Bqd@L(9!j z0MD;*j;1fm;=wBN=+V+XQF~U#HqW)oR~I98KIYz^aclb#-_qHMXTg!uE|G2iUQKz0 z&A4<2Ro>k)ecu|NprwGky0oilzCM=Qt&~6#oT!b|t_%!m(g7G?LWMetO`FclB7i z`7i=XrX5(`2g_c$A!qApefWoP+W?4hUBg|0>q^XD=sag6t~~1FCC{f&n<_YhL)*<= zHziG%0}n-Mun|+6NW~-b<8udt1NH*0Z3uZIbN$tweE7X6iJRHL7ldtlp&Wm;|3G70 zru9w}u0*5E^OQ{WIA_G|;kvcNujB~T2e1AnBAU4#kTwm|VUJ~z_WZ@Ej8u+~vrip! zYU9{EP&n`9P-J_??sSDNZthX(ZrRjg(Sw>-eJ)i9`E7gstI-}AOO!NI{1(M614mv0 zzj2e~GJV%y`l%yvgu()0Y7h+|tN6XRj_*-8GRUPzvb8(bSnw>=_rf;#P{-}Yx|~~+ zUtK(G3%A`z7&zd%D{G$#1GT4W7ryuO)f*6FX)il#x=>Q63`{6D7Z$s1-O9GuR8_x! z&!^LRtQ5uOvB_!o(=(W|3U5J}-0h3#eGs!FphC@NbD^+4BE3?E(^Fq#1$%p8xZ%WB1ox*U9N5#ga+V>KXvV>r~weXv)=v`uz#;RVm=k|qbw~Q-e zw~i&&>>D{pC{(%xWxVx>5s9~}eSgZW8Lx_ZJE*pJzRXEZkTGe`?Y?$qI{4BcIY6QD ze=OhQc>nvh`g!fZD<74}rt(sR*y0&1F~g@~-4t37h8_yzLNdoK_MWWwPK3*_QICW% zVyU&J7Y@mP>&`uu5yq@m8|Pk_w|t)?=i%Izygx_&;ZA-(>)rz~%sJwy1E$jO>vAHN zv<;-ndbp35Z|8ZV@Aswum%+5j<%}A(yXT(N7APypFXOhq)wCOfVRL-;VArd;RRuZv z=!PY{ZFf%sS=)gW1rU^8J9ot-|;f6*Vr(v7#F+hg$w_q#=4 zT@VqP11aaMSE%{rUHk1eLzxHj?|p_vYxD<0s(S#_Ao$GtBPr(yyNvmIxlE<_OWMy2 ziGz-MGI4l`iniijp~wg+WCGqrtFr+h{oH-e>z8!dQ${`4;+OaG#eDBBsBiRe_m0La zM-(k{B}7~}sH{4M=j8MJ9mSh6GJW?V=q@5b#vt@gCuIHVSrkE2tsC=}wI?Y_#@=wA zCF`#WoblKpku~efM(#TbUyXoTT(!_roVNUJbWCE2!N=yrVGbl*r6iTQZxB;hwD-UZUkbM=?w1a2to zy!|C9@={^q%35wA-7Hv~HCnlO8PP6!ACYta^KniZt7b(JDB`r{5`5&S9Hhw2W6j{i;l>Syme4 zo$;+5g|DB^b1Y5V6Tx?m?HC0d=~x>(V<2&uDb-_`v*SYVV!LY(E|T%7HCf?`h|Agi z?FML?e7)hMmi*NxH@Qh-C}bx5Uq)$4!E9{#~!KGP|aEQSioRV-EF3eqO}=`O3pp)VtN9@+|%Q#Jq|*<_9X@ z3Q0u=GrlR34x<2ESIYMX1AVD7(y`cT`UT-={B)3~#rqGkucIo>`X0C_)D(ID!zG_m zR>SmV`?CQfhfQ|B)8~~}GV!Pa=8zi2d+TS?kV`g_LZ(qKA2BzEoc!sjBb_Wd0<-=?O&ThI6gy zx!4lc-#c7BG_iBve-QCcMNNmI08DkSmEQl0Em?T(#71>D_>!z|KG3_lyENYQJ>$mK zLb0-sSpM$k#`0;u@%x%$uZAXh9<+Vqy5xMU_^E)W6XBmlduoy^hgtvDzznS?8hEp?~V^leLQMbbHZpw?XGnBMIxcT>ODFLXYKZb2s_#l zo4B5sx4lLafvqRbQ{4EoQ&ssHq2xC!al$p81MDGXZ3vU1qkHGIjTM@coB%yy^Ap3| z@m)=FpjAIFKfFDTn1}&c<%p&KtF|+bhqC?m_$aL^J!OxOb|Hx=vP3ndXl1g8B7~H6 zFsP{Pl^T&PYu4=RjFd*%*BRRwp{&hVhRF=$e5dDme&;#obeyx0USq!D2 zr#{x1;3B70G5$h(<5mb|b-dx9OD?C& ztCd$Q*I#1IOwjwZo7)?)VLEB?nD&eCr)*+|Oa;GGfr5>>60F_FoF7vn?= z?hRcjZ>Ae9NXiFg2uT^S-Z>{nGLP$R#^~|Qg}k8h_+qWcE-&I`&6jOr$ljlC_a6ua zh*;R}k9$9h7tz;w1SB9+wLnhm&6~Akhn9;IepkkFbL9j0v>D_r=~DKHPE+b36X~;-Aw)=sR`RaFdg& zJk{yQtxnqF{J>-(bC%k#!37( zh|PpR4%iAqajEnO^mT~NI0Xxj4Z$ghe9G8Dc-KCS+sEA;!%04Tn}Hs)h8$|wVsT-_iEy) zl6Liyv0Zc%F2Q={kXELf0z)dp$r|~3)tROK%?cT2xBs}578hT-^HBkeU?F<)>Yk8+ zR?WPrk-jm;XmEVqghy`eR-CWPDJP z=9g=o=a8Z8$OiWxO}h_w=T>$LOYwY$6)ucvD`BZNv-JEI$#D&7lb?i-xRHy6;U@9m z8rsCjLBH^zI#0a7SwU{`YJ$${yO$Ic6wE$%6rcI9%NA$Y9G{3&@}77L{t?CVxuNbD z%ZDx@c-H&#V>s02`RE(>1+JtI8T&I@Z36|Qh_pk#ejDd(ynnuKl^GFm9(ef_Q#z?O zd2t#f@-UPozHo*w7q4G5ad~N%Y8jsPY>HGFCp#jm)$C(mPVdD_x_?C*c8)35uysT7 zS|29TmM@uUQ%A+rul1MCd_b3Gx{ulRCF0ToUz@v3<8ZEINA<44bj{LtWZC-<(b%^> zTRR_i(5^ge?NB4*<>*Bo?(V)R4iW2(0`%3+c&sD&d<}NIv^9z1xByQJkF1kWwOB8q09zV+#oscko1>UA@Ce+Z+f0olceFiI1 zUxF1f>Z#wrIb7uM{f?gJ@vG_z_Sw2gg*v99_RNoK?;9c3Bq5$&D?*+MrO3rRIzeXa z728T13sT!RPHMd(b=UBfcn)UaO{DJzN@mZzj^k#rYs>eF-0SBsN0mA}c!zbouC?fy zzaS6R6(Wugt<^h}jW~V?B-pIe)0}*4c`QSJbV6EoyOTlr1b*;p!k=T^)rxwGvK6zm zZCGR*Qoq%vude9@epb+I8%9-``K-dLJqbzm*w_=f{FtI#IibTAPI*&&m9dmeBENWt zN{G!?{=K$%UZ~p=zr7$KzcQ@?Tc+$8*q~sWlXha|cB{3G2J6k)RYK z9k6Qc*?C>LP0Q08$UHErvxC z7cp)9{8F}`*T?N(eIB$+bfB8Y(~t44!x1v|Q9(ZC<(Xr%`6jb#=RAd^x|};l1BImE zQBGPu23sPXV)}&?!_2+9%!pL*^Kf%pl)Aa@5%3}+RT1G+q}tetM?N*8Ot&&I3&rd7 zX5(aaX^&|9{c^0QgMTk_{c*=Uq@9}+X|%Fw&3(h$wdK-&w(=2TrtFeeqkbnb#U0P2 zTRx>8#6@gY9o7XDjk(=eame%~zcF#o#fJ{_?woa9?P#^}FOfala}VMivR6*xWhG_$ zf_CBuat{8CCpk;|S(nFZkmi&8LdL@$@&`G*KLCtmf67x1v#GvKm&~(#Q(Q!TL^+*2 znT>okVsVR7c_XElywS*O0V}V^YT3w9JO2^lUe0mHQ6gvubR*4po@29B`Mthk<7QKZ z8B?unH(&JLPzwj>t-noj4_Xi5Cgrb$hM|04h20zxmOq#RkzirN+CE)Z7FDtW16myAc3X5EKA8bhYmc=6~;WYF4kt|pGG;+S?R|I2W zj(?mbq^zwn7{;u8$_lDV{0OQX;GRlH*qh9BvoHm@6&_^+OE;=CFl;Y%S6{xa^>TTk zyAtWH4X(!Yji13S z>-nHaOhU`c$+2i+-DIt7LQ9_n@tZE17UbbJ|12pX={)V=zOmVXPmGC5Nv{nv+GnW(@3DfZp>mxTI#RFNfsErc{IgJUH7>AgdKDQJU%IX(TI0z1qV!8}XgU z`lcw%Bu$qNkSp^dI%Pb})Aky)Ym;;uJ@v`fAS|$P}FQRiI-AvK1Ux|tJKH_-~8?&9YRp7IXG~rkF zxNGhnTh>1Du5MqA(&l@m`i6PiL$H3Z3+vF@sQv>VJ!ItHSykS;suskM^z%neP2+L_ z3zU?7pnq%XZfb*P(+&+{t)9g@)ru95RAI)OWqJn*|__#(%0 zJEl6O9>)kTZ zQ@PNV$TyK0njFxTT4bN2d}B6kX@@zilbSJ$72q8)qJs;~)d>{je+rB4WR zmwXIi;`VgwXEz<1gqf37s$0;QLy9&TMP@gVni`*-+td}yoR2o`H9$vP}BVqja_Nix}X+I8Byp`p6ego zxD|a|c`WkXfk{VFRu$D$jWBLDwzgCjpVq5!#DGeXyf-C_-s6UmEh~Dmz_MYGWWmfw z@svy4UWf7l0+dG=sKc(?YNwPO;kj@F#S{T_`u0^BI$Q75-6n)F<-3uE$d1hp_ z!mA7{gdp48J0guv&B$9laF<8TqoguVU zGWxvvCt2?MCdcf*N73rNF9ut{i7UR(xlZx!*^p`Y;m%fEdW`;5rv?2`w;%ZcOT|{=k|J^ zLG$?;CvgT%O^csjG#5u@>)tn{FJQX_K4TqC51*!=(7X__i7{Z2;5|Pm(=yf-k!IIc z@)3#K-c{4Y$M_!oOzglnl{+@A{jg)VV@`xwDN3ES0(C~GoKa3cAjn-^0I^8EO zaNu*1U$W2zFH{S^_o>zNpUE0bjSN)0mw}|7Ny@J3uyx4EQo2T~Y|Dwv0Oq`bGo4zq z{4k+&j7cDU)C(-$|LswZbnwp)X;cQ~ai0CJ|2sQhnc=raTk|5FyrW6i3 zvE)j}_WVQAch(28`Url>FJ;jxV1ZqG1z3xPsh;5mh-3e!i0~;_=jcEEx9aWR_|_So z*FZvB0bzN;WNsTsVId1yk55>haSNpQ|IiOYi5`UQR9AhEje7l>t3~}+Ak=?VDFUey z`u|f&P_zD192xlw(&kdQ-f(Ucor>JzwTFr-xq@hzJ==x0;6B~?TQm6IG?D(`<@8%* zC^f_V2Fxx0hPTDH+qA(lv;)WyM>y+R4$CAnLZGWpjmumS&UDD*QG3}fsA z>tEW5|5QW%LyQRO;r~GwdJbHHdyDJ>G>;75P~w^b126$OVJ(^~cR){zxP-w^f8-08 zPe$$Zs)b45-~m{n;ZQ4*Mv$%kTiojkkH*#!2Lkzo{}T6t4J$v0d!hgSCGK4U$HD;3 z0uUBOgZ$TOyr@H;FBIQ^Q$Q+!_t1Vk4OEdISZy%Z-26+O4EzMBlTqR>k2oB*GYG)7 z4vqZyk;x6Nb3v-Bn;97yf!uuCS0vRZ86wpe!=IJh{JM=R9nRh)e(VyAi-jB+uHN>q z7Kluc^Cb75RJqfA!To=J@aqWYA>F<*FovUWfz+ZeyjK=<4DP^_zaXP955C{x#X`0c_5_c+Clc*k=%229P_0baSW;ObB@}=jxEC|Z|W1$!>bX&E#0Zp zuwAx3G{-jHYj36Ks&mBs8UQie0HRUr=q5>x{!%B`Ub}R=Zyy8U0hAi|l?^IKeS zna|76oT=Ro9mju2UBoo-ZW z;pJ>ywC436+Hw5e%~ns=P>4bdR0R>GPxYA#qobrjwf}gFME9K31}@=op^322>+sO9 z7ZoOU3pmKTR*<}Ta(GM+NvXxDL1*tGtZ=+v%9*IV2lR+jaDP;P&Up8Wr~_J?h(?z3 zct(5TOC}eNVl(MT90JkZQ|XcCgr6rR&J+SiAwCFF{wQN&gcebBCR<4ptZmaI)ZC-b zOZB7xrgF@4I`OyY^yM$;+qWS??Z-9*B1;nctHvX3{bX4Wpi9ftmF0wTUjJ*>jc=6o zBOKmPo*J1WZnPYWE^=EQ8+mb&#C1u9MkPndLZcL3L89jtmbLP|Pn(gq!OQc<73dPn z95$WsR>n1ZidtPw7jQ;g!R5!kU!nSB8Rjs6pHUj-LuiYv$#;2m**g0KIBXU>_NmN? z?aqoiq_sp~O~`_~p6v7*dJEST6rzK)0LxIq@%p#dba#D+hwKKG?muV4!r&4;v)s-j zRV-Sl3B97q$NIx%ltgNJJVtj1(&;FU7a?Lj^yCj2AM1V3Ep&yY)(#JH2Xm#VJer`u zVlZ5}cnv6;xz{v>abTlbVJVwF-jP08C+WBQQDjP(Vz}t=YsFkqTNlOf$DMCa^QESy ze&4}IaVmu&ymOgZy`qfLN4#r}{(kGt%sUvei$q=-h-EzJD#=uKAa#dpl8zleJ9trZ z`KGstALLCs%Zm;G#N!Nv*6;|fucy!!$iam2`=h{$m3qzDaLFy8EV#g7T|s4zCh@*p z=u8X%VPP`^g2Z2&rko~QU_xr#^GS&#G{yZj(jc0OWjAq|#gZg@A8HS!IR-9pF#tZk z*)>2bA&J!OKlH>UTe}iiM%cN6L%)(Yvj++c(6E))=lhhl?70CL+ndHT3&^Nsr%b#+`j~BwSEc?;(~rvp=X%J}8fb4#(rMar**L`q?lLxs1MRDhQILKV z^&?8b0?bY4?-e~?C#5V}Xldc~U64A{o;|DsMcJsX@+DGQ=Yf;8aPkwm3zm~M#2%rI z)|AwJrW%VQ+o55QVQmL>3LV}f$R!hjA|!TC9}K;$-9sTs4l5mOglinP<)OQ>51$DD z;?6I0^(i6Zv5+8&V~!W24F$-YlDp3P${gjums`%1--Q1cr2Ec4Y#i6`~K1ZRQ2b+Hjf1Rk-$ifD=m5|7=@q;y+{}iJpflxnHLTDw-EUCAp^VKsv5o zyXie1x*$`}GL$0tcW2{H^uGaFGbUWAdj^_g@~!K*y-i%H4tm?v0KU((cOYo0dtHu*lLa31GQK3`E{3Ddr2cWXj7&^77(?IEmKC9LbsJ1Cb~%KIFo_& zp?*~ieM>y{`6u9pjj5N#q9$Oq%6&y`5s>=*gVcBKN()~8QC#eV`?|tic5z>4z5f2v c=W{FkJtn>daRNuPxSyOmt$!+4`^JO+06E23NB{r; literal 0 HcmV?d00001 From 751b9ceb929dc1165472fb65cba35e04c5a0f674 Mon Sep 17 00:00:00 2001 From: Ravin Kohli <13005107+ravinkohli@users.noreply.github.com> Date: Mon, 22 Nov 2021 14:30:55 +0100 Subject: [PATCH 06/10] [FIX] Silence catboost (#338) * set verbose=False in catboost * fix flake --- .../setup/traditional_ml/traditional_learner/learners.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/autoPyTorch/pipeline/components/setup/traditional_ml/traditional_learner/learners.py b/autoPyTorch/pipeline/components/setup/traditional_ml/traditional_learner/learners.py index eeb1094f1..220c52dcd 100644 --- a/autoPyTorch/pipeline/components/setup/traditional_ml/traditional_learner/learners.py +++ b/autoPyTorch/pipeline/components/setup/traditional_ml/traditional_learner/learners.py @@ -135,7 +135,11 @@ def _fit(self, X_train: np.ndarray, X_train_pooled = Pool(data=X_train, label=y_train, cat_features=categoricals) X_val_pooled = Pool(data=X_val, label=y_val, cat_features=categoricals) - self.model.fit(X_train_pooled, eval_set=X_val_pooled, use_best_model=True, early_stopping_rounds=early_stopping) + self.model.fit(X_train_pooled, + eval_set=X_val_pooled, + use_best_model=True, + early_stopping_rounds=early_stopping, + verbose=False) @staticmethod def get_properties( From 9d93085f746684e3523e45e477623a978dae21a2 Mon Sep 17 00:00:00 2001 From: Ravin Kohli <13005107+ravinkohli@users.noreply.github.com> Date: Mon, 22 Nov 2021 18:56:41 +0100 Subject: [PATCH 07/10] change worst possible result of r2 (#340) --- autoPyTorch/pipeline/components/training/metrics/metrics.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/autoPyTorch/pipeline/components/training/metrics/metrics.py b/autoPyTorch/pipeline/components/training/metrics/metrics.py index b669e4ede..0d82b9622 100644 --- a/autoPyTorch/pipeline/components/training/metrics/metrics.py +++ b/autoPyTorch/pipeline/components/training/metrics/metrics.py @@ -36,7 +36,8 @@ worst_possible_result=MAXINT, greater_is_better=False) r2 = make_metric('r2', - sklearn.metrics.r2_score) + sklearn.metrics.r2_score, + worst_possible_result=-MAXINT) # Standard Classification Scores accuracy = make_metric('accuracy', From f7e9cf8d98cfb97be37629650624f082b3317b51 Mon Sep 17 00:00:00 2001 From: Ravin Kohli <13005107+ravinkohli@users.noreply.github.com> Date: Mon, 22 Nov 2021 19:12:55 +0100 Subject: [PATCH 08/10] Update README.md with link for master branch --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a1818caba..389ab7902 100755 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ While early AutoML frameworks focused on optimizing traditional ML pipelines and Auto-PyTorch is mainly developed to support tabular data (classification, regression). The newest features in Auto-PyTorch for tabular data are described in the paper ["Auto-PyTorch Tabular: Multi-Fidelity MetaLearning for Efficient and Robust AutoDL"](https://arxiv.org/abs/2006.13799) (see below for bibtex ref). -Also, find the documentation [here](https://automl.github.io/Auto-PyTorch/development). +Also, find the documentation [here](https://automl.github.io/Auto-PyTorch/master). ***From v0.1.0, AutoPyTorch has been updated to further improve usability, robustness and efficiency by using SMAC as the underlying optimization package as well as changing the code structure. Therefore, moving from v0.0.2 to v0.1.0 will break compatibility. In case you would like to use the old API, you can find it at [`master_old`](https://github.com/automl/Auto-PyTorch/tree/master-old).*** From 5b65244a3da9047112b9634546587f2cbdcdf463 Mon Sep 17 00:00:00 2001 From: Ravin Kohli <13005107+ravinkohli@users.noreply.github.com> Date: Mon, 22 Nov 2021 22:55:01 +0100 Subject: [PATCH 09/10] [FIX formatting in docs (#342) * fix formatting in docs * Update examples/40_advanced/example_resampling_strategy.py --- .../example_custom_configuration_space.py | 153 +++++++++--------- .../40_advanced/example_parallel_n_jobs.py | 11 +- .../example_resampling_strategy.py | 30 ++-- .../40_advanced/example_run_with_portfolio.py | 84 +++++----- 4 files changed, 144 insertions(+), 134 deletions(-) diff --git a/examples/40_advanced/example_custom_configuration_space.py b/examples/40_advanced/example_custom_configuration_space.py index c64a4fca1..985d9d9ff 100644 --- a/examples/40_advanced/example_custom_configuration_space.py +++ b/examples/40_advanced/example_custom_configuration_space.py @@ -5,6 +5,7 @@ The following example shows how adjust the configuration space of the search. Currently, there are two changes that can be made to the space:- + 1. Adjust individual hyperparameters in the pipeline 2. Include or exclude components: a) include: Dictionary containing components to include. Key is the node @@ -57,80 +58,78 @@ def get_search_space_updates(): return updates -if __name__ == '__main__': - - ############################################################################ - # Data Loading - # ============ - X, y = sklearn.datasets.fetch_openml(data_id=40981, return_X_y=True, as_frame=True) - X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split( - X, - y, - random_state=1, - ) - - ############################################################################ - # Build and fit a classifier with include components - # ================================================== - api = TabularClassificationTask( - search_space_updates=get_search_space_updates(), - include_components={'network_backbone': ['MLPBackbone', 'ResNetBackbone'], - 'encoder': ['OneHotEncoder']} - ) - - ############################################################################ - # Search for an ensemble of machine learning algorithms - # ===================================================== - api.search( - X_train=X_train.copy(), - y_train=y_train.copy(), - X_test=X_test.copy(), - y_test=y_test.copy(), - optimize_metric='accuracy', - total_walltime_limit=150, - func_eval_time_limit_secs=30 - ) - - ############################################################################ - # Print the final ensemble performance - # ==================================== - y_pred = api.predict(X_test) - score = api.score(y_pred, y_test) - print(score) - print(api.show_models()) - - # Print statistics from search - print(api.sprint_statistics()) - - ############################################################################ - # Build and fit a classifier with exclude components - # ================================================== - api = TabularClassificationTask( - search_space_updates=get_search_space_updates(), - exclude_components={'network_backbone': ['MLPBackbone'], - 'encoder': ['OneHotEncoder']} - ) - - ############################################################################ - # Search for an ensemble of machine learning algorithms - # ===================================================== - api.search( - X_train=X_train, - y_train=y_train, - X_test=X_test.copy(), - y_test=y_test.copy(), - optimize_metric='accuracy', - total_walltime_limit=150, - func_eval_time_limit_secs=30 - ) - - ############################################################################ - # Print the final ensemble performance - # ==================================== - y_pred = api.predict(X_test) - score = api.score(y_pred, y_test) - print(score) - print(api.show_models()) - - # Print statistics from search - print(api.sprint_statistics()) +############################################################################ +# Data Loading +# ============ +X, y = sklearn.datasets.fetch_openml(data_id=40981, return_X_y=True, as_frame=True) +X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split( + X, + y, + random_state=1, +) + +############################################################################ +# Build and fit a classifier with include components +# ================================================== +api = TabularClassificationTask( + search_space_updates=get_search_space_updates(), + include_components={'network_backbone': ['MLPBackbone', 'ResNetBackbone'], + 'encoder': ['OneHotEncoder']} +) + +############################################################################ +# Search for an ensemble of machine learning algorithms +# ===================================================== +api.search( + X_train=X_train.copy(), + y_train=y_train.copy(), + X_test=X_test.copy(), + y_test=y_test.copy(), + optimize_metric='accuracy', + total_walltime_limit=150, + func_eval_time_limit_secs=30 +) + +############################################################################ +# Print the final ensemble performance +# ==================================== +y_pred = api.predict(X_test) +score = api.score(y_pred, y_test) +print(score) +print(api.show_models()) + +# Print statistics from search +print(api.sprint_statistics()) + +############################################################################ +# Build and fit a classifier with exclude components +# ================================================== +api = TabularClassificationTask( + search_space_updates=get_search_space_updates(), + exclude_components={'network_backbone': ['MLPBackbone'], + 'encoder': ['OneHotEncoder']} +) + +############################################################################ +# Search for an ensemble of machine learning algorithms +# ===================================================== +api.search( + X_train=X_train, + y_train=y_train, + X_test=X_test.copy(), + y_test=y_test.copy(), + optimize_metric='accuracy', + total_walltime_limit=150, + func_eval_time_limit_secs=30 +) + +############################################################################ +# Print the final ensemble performance +# ==================================== +y_pred = api.predict(X_test) +score = api.score(y_pred, y_test) +print(score) +print(api.show_models()) + +# Print statistics from search +print(api.sprint_statistics()) diff --git a/examples/40_advanced/example_parallel_n_jobs.py b/examples/40_advanced/example_parallel_n_jobs.py index 698f3ad61..d345c6fca 100644 --- a/examples/40_advanced/example_parallel_n_jobs.py +++ b/examples/40_advanced/example_parallel_n_jobs.py @@ -1,10 +1,11 @@ """ -====================== -Tabular Classification -====================== +============================================ +Tabular Classification with n parallel jobs +============================================ The following example shows how to fit a sample classification model parallely on 2 cores with AutoPyTorch + """ import os import tempfile as tmp @@ -60,9 +61,9 @@ ############################################################################ # Print the final ensemble performance # ==================================== - print(api.run_history, api.trajectory) y_pred = api.predict(X_test) score = api.score(y_pred, y_test) print(score) # Print the final ensemble built by AutoPyTorch - print(api.show_models()) + print(api.sprint_statistics()) + diff --git a/examples/40_advanced/example_resampling_strategy.py b/examples/40_advanced/example_resampling_strategy.py index 6735fffee..d02859f1b 100644 --- a/examples/40_advanced/example_resampling_strategy.py +++ b/examples/40_advanced/example_resampling_strategy.py @@ -26,10 +26,13 @@ from autoPyTorch.api.tabular_classification import TabularClassificationTask from autoPyTorch.datasets.resampling_strategy import CrossValTypes, HoldoutValTypes +############################################################################ +# Default Resampling Strategy +# ============================ ############################################################################ # Data Loading -# ============ +# ------------ X, y = sklearn.datasets.fetch_openml(data_id=40981, return_X_y=True, as_frame=True) X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split( X, @@ -39,7 +42,7 @@ ############################################################################ # Build and fit a classifier with default resampling strategy -# =========================================================== +# ----------------------------------------------------------- api = TabularClassificationTask( # 'HoldoutValTypes.holdout_validation' with 'val_share': 0.33 # is the default argument setting for TabularClassificationTask. @@ -51,7 +54,7 @@ ############################################################################ # Search for an ensemble of machine learning algorithms -# ===================================================== +# ----------------------------------------------------- api.search( X_train=X_train, y_train=y_train, @@ -64,7 +67,7 @@ ############################################################################ # Print the final ensemble performance -# ==================================== +# ------------------------------------ y_pred = api.predict(X_test) score = api.score(y_pred, y_test) print(score) @@ -76,9 +79,13 @@ ############################################################################ +############################################################################ +# Cross validation Resampling Strategy +# ===================================== + ############################################################################ # Build and fit a classifier with Cross validation resampling strategy -# ==================================================================== +# -------------------------------------------------------------------- api = TabularClassificationTask( resampling_strategy=CrossValTypes.k_fold_cross_validation, resampling_strategy_args={'num_splits': 3} @@ -86,7 +93,8 @@ ############################################################################ # Search for an ensemble of machine learning algorithms -# ===================================================== +# ----------------------------------------------------------------------- + api.search( X_train=X_train, y_train=y_train, @@ -99,7 +107,7 @@ ############################################################################ # Print the final ensemble performance -# ==================================== +# ------------ y_pred = api.predict(X_test) score = api.score(y_pred, y_test) print(score) @@ -111,9 +119,13 @@ ############################################################################ +############################################################################ +# Stratified Resampling Strategy +# =============================== + ############################################################################ # Build and fit a classifier with Stratified resampling strategy -# ============================================================== +# -------------------------------------------------------------- api = TabularClassificationTask( # For demonstration purposes, we use # Stratified hold out validation. However, @@ -124,7 +136,7 @@ ############################################################################ # Search for an ensemble of machine learning algorithms -# ===================================================== +# ----------------------------------------------------- api.search( X_train=X_train, y_train=y_train, diff --git a/examples/40_advanced/example_run_with_portfolio.py b/examples/40_advanced/example_run_with_portfolio.py index 01d8bef15..fef230fc5 100644 --- a/examples/40_advanced/example_run_with_portfolio.py +++ b/examples/40_advanced/example_run_with_portfolio.py @@ -24,50 +24,48 @@ from autoPyTorch.api.tabular_classification import TabularClassificationTask -if __name__ == '__main__': +############################################################################ +# Data Loading +# ============ +X, y = sklearn.datasets.fetch_openml(data_id=40981, return_X_y=True, as_frame=True) +X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split( + X, + y, + random_state=42, +) - ############################################################################ - # Data Loading - # ============ - X, y = sklearn.datasets.fetch_openml(data_id=40981, return_X_y=True, as_frame=True) - X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split( - X, - y, - random_state=42, - ) +############################################################################ +# Build and fit a classifier +# ========================== +api = TabularClassificationTask( + seed=42, +) - ############################################################################ - # Build and fit a classifier - # ========================== - api = TabularClassificationTask( - seed=42, - ) +############################################################################ +# Search for an ensemble of machine learning algorithms +# ===================================================== +api.search( + X_train=X_train, + y_train=y_train, + X_test=X_test.copy(), + y_test=y_test.copy(), + optimize_metric='accuracy', + total_walltime_limit=300, + func_eval_time_limit_secs=50, + # Setting this option to "greedy" + # will make smac run the configurations + # present in 'autoPyTorch/configs/greedy_portfolio.json' + portfolio_selection="greedy" +) - ############################################################################ - # Search for an ensemble of machine learning algorithms - # ===================================================== - api.search( - X_train=X_train, - y_train=y_train, - X_test=X_test.copy(), - y_test=y_test.copy(), - optimize_metric='accuracy', - total_walltime_limit=300, - func_eval_time_limit_secs=50, - # Setting this option to "greedy" - # will make smac run the configurations - # present in 'autoPyTorch/configs/greedy_portfolio.json' - portfolio_selection="greedy" - ) +############################################################################ +# Print the final ensemble performance +# ==================================== +y_pred = api.predict(X_test) +score = api.score(y_pred, y_test) +print(score) +# Print the final ensemble built by AutoPyTorch +print(api.show_models()) - ############################################################################ - # Print the final ensemble performance - # ==================================== - y_pred = api.predict(X_test) - score = api.score(y_pred, y_test) - print(score) - # Print the final ensemble built by AutoPyTorch - print(api.show_models()) - - # Print statistics from search - print(api.sprint_statistics()) +# Print statistics from search +print(api.sprint_statistics()) From ef8d21ae982883548a8e77a3690eefa810d3ae07 Mon Sep 17 00:00:00 2001 From: Ravin Kohli <13005107+ravinkohli@users.noreply.github.com> Date: Tue, 23 Nov 2021 11:14:57 +0100 Subject: [PATCH 10/10] Update README.md, remove cat requirements.txt --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 389ab7902..3fb449fb1 100755 --- a/README.md +++ b/README.md @@ -56,7 +56,6 @@ git submodule update --init --recursive conda create -n auto-pytorch python=3.8 conda activate auto-pytorch conda install swig -cat requirements.txt | xargs -n 1 -L 1 pip install python setup.py install ```