Merge branch 'feature/sawei' of github.com:automl/SMAC3 into feature/sawei

Author: benjamc

examples/6_advanced_features/2_SAWEI.py

@@ -18,7 +18,9 @@
 
 import smac
 import numpy as np
-from smac.callback.sawei_callback import get_sawei_kwargs
+import pandas as pd
+import seaborn as sns
+from smac.callback.sawei_callback import get_sawei_kwargs, WEITracker, UpperBoundRegretCallback
 
 
 __copyright__ = "Copyright 2021, AutoML.org Freiburg-Hannover"
@@ -28,6 +30,11 @@
 class QuadraticFunction:
     @property
     def configspace(self) -> ConfigurationSpace:
+        """Configuration/search space of the problem
+
+        Returns:
+            ConfigurationSpace: Configuration space.
+        """
         cs = ConfigurationSpace(seed=0)
         x = Float("x", (-5, 5), default=-5)
         cs.add_hyperparameters([x])
@@ -41,6 +48,14 @@ def train(self, config: Configuration, seed: int = 0) -> float:
 
 
 def plot(runhistory: RunHistory, incumbent: Configuration) -> None:
+    """Plot the objective function, all trials and the incumbent
+
+    Args:
+        runhistory (RunHistory): Runhistory object after optimization
+        incumbent (Configuration): Best configuration found by SMAC
+    """
+    sns.set_style("whitegrid")
+    sns.set_palette("colorblind")
     plt.figure()
 
     # Plot ground truth
@@ -60,34 +75,72 @@ def plot(runhistory: RunHistory, incumbent: Configuration) -> None:
 
     plt.show()
 
+def plot_sawei(facade: smac.facade.abstract_facade.AbstractFacade) -> None:
+    """Plot the interpolation weight alpha of SAWEI and the UBR (Upper Bound Regret)
+
+    Args:
+        facade (smac.facade.abstract_facade.AbstractFacade): SMAC facade object after optimization
+    """
+
+    sns.set_style("whitegrid")
+    sns.set_palette("colorblind")
+    
+    weitracker = None
+    ubrtracker = None
+    for callback in facade._callbacks:
+        if isinstance(callback, WEITracker):
+            weitracker = callback
+        if isinstance(callback, UpperBoundRegretCallback):
+            ubrtracker = callback
+
+    # The data also lies in the output folder as a csv file
+    df_alpha = pd.DataFrame(weitracker.history)
+    df_ubr = pd.DataFrame(ubrtracker.history)
+    df = pd.merge(df_alpha, df_ubr, on="n_evaluated")
+
+    fig, ax = plt.subplots()
+    ax2 = ax.twinx()
+    ax = sns.lineplot(data=df, x="n_evaluated", y="alpha", ax=ax, label="alpha", color=sns.color_palette()[0])
+    ax2 = sns.lineplot(data=df, x="n_evaluated", y="ubr", ax=ax2, label="ubr", color=sns.color_palette()[1])
+    ax.legend(loc="upper left")
+    ax2.legend(loc="upper right")
+    ax.set_ylabel("alpha")
+    ax2.set_ylabel("ubr")
+    ax.set_xlabel("n_evaluated")
+    plt.show()
+    
+    
 
 if __name__ == "__main__":
     model = QuadraticFunction()
 
     # Scenario object specifying the optimization "environment"
-    scenario = Scenario(model.configspace, deterministic=True, n_trials=100, seed=np.random.randint(low=0,high=10000))
+    scenario = Scenario(model.configspace, deterministic=True, n_trials=50, seed=np.random.randint(low=0,high=10000))
 
     # Get the kwargs necessary to use SAWEI
     # SAWEI is implemented as a chain of callbacks
     sawei_kwargs = get_sawei_kwargs()
 
     # Now we use SMAC to find the best hyperparameters
-    smac = BlackBoxFacade(
+    optimizer = BlackBoxFacade(
         scenario,
         model.train,  # We pass the target function here
         overwrite=True,  # Overrides any previous results that are found that are inconsistent with the meta-data
         **sawei_kwargs  # Add SAWEI
     )
 
-    incumbent = smac.optimize()
+    incumbent = optimizer.optimize()
 
     # Get cost of default configuration
-    default_cost = smac.validate(model.configspace.get_default_configuration())
+    default_cost = optimizer.validate(model.configspace.get_default_configuration())
     print(f"Default cost: {default_cost}")
 
     # Let's calculate the cost of the incumbent
-    incumbent_cost = smac.validate(incumbent)
+    incumbent_cost = optimizer.validate(incumbent)
     print(f"Incumbent cost: {incumbent_cost}")
 
     # Let's plot it too
-    plot(smac.runhistory, incumbent)
+    plot(optimizer.runhistory, incumbent)
+
+    # Let's plot the interpolation weight alpha of SAWEI and the UBR (Upper Bound Regret)
+    plot_sawei(optimizer)

setup.py

@@ -59,6 +59,9 @@ def read_file(filepath: str) -> str:
         "pytest-coverage",
         "pytest-cases",
     ],
+    "wandb": [
+        "wandb",
+    ]
 }
 
 setuptools.setup(
@@ -88,6 +91,7 @@ def read_file(filepath: str) -> str:
         "emcee>=3.0.0",
         "regex",
         "pyyaml",
+        "seaborn",
     ],
     extras_require=extras_require,
     test_suite="pytest",

smac/callback/utils.py

@@ -2,16 +2,18 @@
 import numpy as np
 
 
-def query_callback(solver: smac.main.smbo.SMBO, callback_type: str, key: str) -> float:
+def query_callback(solver: smac.main.smbo.SMBO, callback_type: str, key: str, raise_error: bool = True) -> float:
     obs = None
     for callback in solver._callbacks:
         if type(callback).__name__ == callback_type:
             if callback.history:
                 obs = callback.history[-1][key]
+            elif hasattr(callback, key):
+                obs = getattr(callback, key)
             else:
                 obs = -np.inf  # FIXME: alpha can only take 0-1 so -np.inf is not too correct
             break
-    if obs is None:
+    if obs is None and raise_error:
         raise ValueError(f"Couldn't find the {callback_type} callback.")
 
     return obs

smac/callback/wandb_callback.py

@@ -0,0 +1,54 @@
+from __future__ import annotations
+
+import smac
+from smac.callback import Callback
+
+
+class WandBCallback(Callback):
+    def __init__(
+            self,
+            project: str,
+            entity: str,
+            id: str | None = None,
+            outdir: str | None = None,
+            mode: str | None = None,
+            resume: str = "allow",
+            job_type: str | None = None,
+            group: str | None = None,
+            config: dict | str | None = None,
+            save_code: bool = True,
+            **kwargs
+        ) -> None:
+        import wandb
+        self.run = wandb.init(
+            id=id,
+            resume=resume,
+            mode=mode,
+            project=project,
+            job_type=job_type,
+            entity=entity,
+            group=group,
+            dir=outdir,
+            config=config,
+            save_code=save_code,
+            **kwargs
+        )
+        super().__init__()
+
+
+    def on_end(self, smbo: smac.main.smbo.SMBO) -> None:
+        intensifier_data = smbo.intensifier.get_data()
+        trajectory = intensifier_data["trajectory"]
+        import pandas as pd
+        df = pd.DataFrame(data=trajectory)
+        print(df)
+        # trajectory = Table(dataframe=df, allow_mixed_types=True)
+        df["costs"] = df["costs"].apply(lambda x: x[0])  # TODO properly log multi costs
+        for index, row in df.iterrows():
+            print(dict(row))
+            self.run.log(dict(row))
+        self.run.finish()
+        return super().on_end(smbo)
+
+
+

smac/intensifier/intensifier.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from typing import Any, Iterator
+import dataclasses
 
 from ConfigSpace import Configuration
 
@@ -90,6 +91,16 @@ def get_state(self) -> dict[str, Any]:  # noqa: D102
 
     def set_state(self, state: dict[str, Any]) -> None:  # noqa: D102
         self._queue = [(self.runhistory.get_config(id), n) for id, n in state["queue"]]
+    
+    def get_data(self):
+        data = {
+            "incumbent_ids": [self.runhistory.get_config_id(config) for config in self._incumbents],
+            "rejected_config_ids": self._rejected_config_ids,
+            "incumbents_changed": self._incumbents_changed,
+            "trajectory": [dataclasses.asdict(item) for item in self._trajectory],
+            "state": self.get_state(),
+        }
+        return data
 
     def __iter__(self) -> Iterator[TrialInfo]:
         """This iter method holds the logic for the intensification loop.

smac/main/config_selector.py

@@ -217,11 +217,13 @@ def __iter__(self) -> Iterator[Configuration]:
             # we don't need to train the next time
             self._previous_entries = Y.shape[0]
 
-            # Now we maximize the acquisition function
+            # Now we maximize the acquisition function            
             challengers = self._acquisition_maximizer.maximize(
                 previous_configs,
                 random_design=self._random_design,
             )
+            # DO NOT FIDDLE WITH THE CHALLENGERS LIST because it is an iterator
+            # e.g. if we convert it to a list, the following for loop does not work!
 
             counter = 0
             failed_counter = 0
@@ -241,6 +243,10 @@ def __iter__(self) -> Iterator[Configuration]:
                         )
                         break
                 else:
+                    logger.debug(
+                            f"Config {config} was already processed. Increase fail counter. "\
+                             "Re-maximize acquisition function to find a new config."
+                        )
                     failed_counter += 1
 
                     # We exit the loop if we have tried to add the same configuration too often