Merge pull request #95 from automl/development

Prepare release 1.0.5

Author: shukon

pimp/__version__.py

@@ -1,2 +1,2 @@
 """Version information."""
-__version__ = "1.0.4"
+__version__ = "1.0.5"

pimp/epm/base_epm.py

@@ -79,7 +79,11 @@ def __init__(self, types: np.ndarray,
         logged_y: bool
             Indicates if the y data is transformed (i.e. put on logscale) or not
         """
-        super().__init__(**kwargs)
+        try:
+            super().__init__(types, bounds, **kwargs)
+        except TypeError:
+            # To ensure backwards-compatibility with smac<0.9.0
+            super().__init__(**kwargs)
 
         self.types = types
         self.bounds = bounds

pimp/evaluator/fanova.py

@@ -11,8 +11,9 @@
 from matplotlib import pyplot as plt
 
 from smac.runhistory.runhistory import RunHistory
+from ConfigSpace.configuration_space import ConfigurationSpace, Configuration
 from ConfigSpace.util import impute_inactive_values
-from ConfigSpace.hyperparameters import CategoricalHyperparameter
+from ConfigSpace.hyperparameters import CategoricalHyperparameter, Constant
 
 try:
     from fanova import fANOVA as fanova_pyrfr
@@ -40,6 +41,17 @@ def __init__(self, scenario, cs, model, to_evaluate: int, runhist: RunHistory, r
         super().__init__(scenario, cs, model, to_evaluate, rng, **kwargs)
         self.name = 'fANOVA'
         self.logger = 'pimp.' + self.name
+
+        # Turn all Constants into Categoricals (fANOVA cannot handle Constants)
+        self.cs_contained_constant = False
+        if any([isinstance(hp, Constant) for hp in self.cs.get_hyperparameters()]):
+            self.logger.debug("Replacing configspace's hyperparameter Constants by one-value Categoricals.")
+            new_hyperparameters = [CategoricalHyperparameter(hp.name, [hp.value]) if isinstance(hp, Constant)
+                                   else hp for hp in self.cs.get_hyperparameters()]
+            self.cs = ConfigurationSpace()
+            self.cs.add_hyperparameters(new_hyperparameters)
+            self.cs_contained_constant = True
+
         # This way the instance features in X are ignored and a new forest is constructed
         if self.model.instance_features is None:
             self.logger.info('No preprocessing necessary')
@@ -59,7 +71,7 @@ def __init__(self, scenario, cs, model, to_evaluate: int, runhist: RunHistory, r
             cutoffs = (self.model.predict_marginalized_over_instances(
                 np.array([impute_inactive_values( self.cs.get_default_configuration()).get_array()]))[0].flatten()[0],
                        np.inf)
-        self.evaluator = fanova_pyrfr(X=self.X, Y=self.y.flatten(), config_space=cs,
+        self.evaluator = fanova_pyrfr(X=self.X, Y=self.y.flatten(), config_space=self.cs,
                                       seed=self.rng.randint(2**31-1), cutoffs=cutoffs)
         self.n_most_imp_pairs = n_pairs
         self.num_single = None
@@ -77,13 +89,16 @@ def _preprocess(self, runhistory):
         self.logger.info('PREPROCESSING PREPROCESSING PREPROCESSING PREPROCESSING PREPROCESSING PREPROCESSING')
         self.logger.info('Marginalizing away all instances!')
         configs = runhistory.get_all_configs()
+        if self.cs_contained_constant:
+            configs = [Configuration(self.cs, vector=c.get_array()) for c in configs]
         X_non_hyper, X_prime = [], []
         for config in configs:
             config = impute_inactive_values(config).get_array()
             X_prime.append(config)
             X_non_hyper.append(config)
             for idx, param in enumerate(self.cs.get_hyperparameters()):
-                if not isinstance(param, CategoricalHyperparameter):
+                if not (isinstance(param, CategoricalHyperparameter) or
+                        isinstance(param, Constant)):
                     X_non_hyper[-1][idx] = param._transform(X_non_hyper[-1][idx])
         X_non_hyper = np.array(X_non_hyper)
         X_prime = np.array(X_prime)

pimp/evaluator/local_parameter_importance.py

@@ -41,6 +41,7 @@ def __init__(self, scenario, cs, model, to_evaluate: int, incumbent=None, contin
         self._sampled_neighbors = 0
         self.variance_dict = {}
         self.quantify_importance_via_variance = quant_var
+        self.evaluated_parameter_importance_uncertainty = OrderedDict()
 
     def _old_sampling_of_one_exchange_neighborhood(self, param, array, index):
         neighbourhood = []
@@ -163,6 +164,7 @@ def _get_one_exchange_neighborhood_by_parameter(self):
     def run(self) -> OrderedDict:
         """
         Main function.
+
         Returns
         -------
         evaluated_parameter_importance:OrderedDict
@@ -182,14 +184,15 @@ def run(self) -> OrderedDict:
         pbar = tqdm(range(self._sampled_neighbors), ascii=True, disable=not self.verbose)
         sum_var = 0
         for index, param in enumerate(self.incumbent.keys()):
+            # Iterate over parameters
             if param in neighborhood_dict:
                 pbar.set_description('Predicting performances for neighbors of {: >.30s}'.format(param))
                 performance_dict[param] = []
                 variance_dict[param] = []
-                variance_dict[param] = []
                 overall_var[param] = []
                 added_inc = False
                 inc_at = 0
+                # Iterate over neighbors
                 for unit_neighbor, neighbor in zip(neighborhood_dict[param][0], neighborhood_dict[param][1]):
                     if not added_inc:
                         if unit_neighbor > incumbent_array[index]:
@@ -262,6 +265,14 @@ def run(self) -> OrderedDict:
         self.performance_dict = performance_dict
         self.variance_dict = variance_dict
         self.evaluated_parameter_importance = OrderedDict(tmp)
+        # Estimate uncertainty using the law of total variance
+        for param in self.evaluated_parameter_importance.keys():
+            mean_over_vars = np.mean(variance_dict[param])
+            var_over_means = np.var(performance_dict[param])
+            # self.logger.debug("vars=%s, means=%s", str(variance_dict[param]), str(performance_dict[param]))
+            self.logger.debug("Using law of total variance yields for %s: mean_over_vars=%f, var_over_means=%f (sum=%f)", param,
+                              mean_over_vars, var_over_means, mean_over_vars + var_over_means)
+            self.evaluated_parameter_importance_uncertainty[param] = mean_over_vars + var_over_means
         all_res = {'imp': self.evaluated_parameter_importance,
                    'order': list(self.evaluated_parameter_importance.keys())}
         return all_res