Merge pull request #136 from automl/development

Development

README.md

@@ -2,39 +2,50 @@
 
 HPOBench is a library for providing benchmarks for (multi-fidelity) hyperparameter optimization and with a focus on reproducibility.
 
-A list of benchmarks can be found in the [wiki](https://github.com/automl/HPOBench/wiki/Available-Containerized-Benchmarks) and a guide on howto contribute benchmarks is avaiable [here](https://github.com/automl/HPOBench/wiki/How-to-add-a-new-benchmark-step-by-step)
+Further info:
+  * list of [benchmarks](https://github.com/automl/HPOBench/wiki/Available-Containerized-Benchmarks)
+  * [howto](https://github.com/automl/HPOBench/wiki/How-to-add-a-new-benchmark-step-by-step) contribute benchmarks
 
 ## Status
 
 Status for Master Branch: 
-[![Build Status](https://github.com/automl/HPOBench/workflows/Test%20Pull%20Requests/badge.svg?branch=master)](https://https://github.com/automl/HPOBench/actions)
+[![Build Status](https://github.com/automl/HPOBench/workflows/Test%20Pull%20Requests/badge.svg?branch=master)](https://github.com/automl/HPOBench/actions)
 [![codecov](https://codecov.io/gh/automl/HPOBench/branch/master/graph/badge.svg)](https://codecov.io/gh/automl/HPOBench)
 
 Status for Development Branch: 
-[![Build Status](https://github.com/automl/HPOBench/workflows/Test%20Pull%20Requests/badge.svg?branch=development)](https://https://github.com/automl/HPOBench/actions)
+[![Build Status](https://github.com/automl/HPOBench/workflows/Test%20Pull%20Requests/badge.svg?branch=development)](https://github.com/automl/HPOBench/actions)
 [![codecov](https://codecov.io/gh/automl/HPOBench/branch/development/graph/badge.svg)](https://codecov.io/gh/automl/HPOBench)
 
 ## In 4 lines of code
 
 Evaluate a random configuration using a singularity container
 ```python
-from hpobench.container.benchmarks.ml.xgboost_benchmark import XGBoostBenchmark
-b = XGBoostBenchmark(task_id=167149, container_source='library://phmueller/automl', rng=1)
+from hpobench.container.benchmarks.nas.tabular_benchmarks import SliceLocalizationBenchmark
+b = SliceLocalizationBenchmark(rng=1)
 config = b.get_configuration_space(seed=1).sample_configuration()
-result_dict = b.objective_function(configuration=config, fidelity={"n_estimators": 128, "dataset_fraction": 0.5}, rng=1)
+result_dict = b.objective_function(configuration=config, fidelity={"budget": 100}, rng=1)
 ```
 
 All benchmarks can also be queried with fewer or no fidelities:
 
 ```python
-from hpobench.container.benchmarks.ml.xgboost_benchmark import XGBoostBenchmark
-b = XGBoostBenchmark(task_id=167149, container_source='library://phmueller/automl', rng=1)
+from hpobench.container.benchmarks.nas.tabular_benchmarks import SliceLocalizationBenchmark
+b = SliceLocalizationBenchmark(rng=1)
 config = b.get_configuration_space(seed=1).sample_configuration()
-result_dict = b.objective_function(configuration=config, fidelity={"n_estimators": 128,}, rng=1)
+result_dict = b.objective_function(configuration=config, fidelity={"budget": 50}, rng=1)
+# returns results on the highest budget
 result_dict = b.objective_function(configuration=config, rng=1)
 ```
 
-For more examples see `/example/`.
+For each benchmark further info on the searchspace and fidelity space can be obtained:
+
+```python
+from hpobench.container.benchmarks.nas.tabular_benchmarks import SliceLocalizationBenchmark
+b = SliceLocalizationBenchmark(task_id=167149, rng=1)
+cs = b.get_configuration_space(seed=1)
+fs = b.get_fidelity_space(seed=1)
+meta = b.get_meta_information()
+```
 
 ## Installation
 

changelog.md

@@ -1,3 +1,12 @@
+# 0.0.10
+  * Cartpole Benchmark Version 0.0.4:
+    Fix: Pass the hp `entropy_regularization` to the PPO Agent. 
+    Increase the lower limit of `likelihood_ratio_clipping` from 0 to 10e-7 (0 is invalid.)
+  * Tabular Benchmark (NAS) Version 0.0.5 + NAS201 Version 0.0.5:
+    We add for each benchmark in nas/tabular_benchmarks and nas/nasbench_201 a new benchmark class with a modified fidelity space. The new benchmark are called _Original_, e.g. _SliceLocalizationBenchmarkOriginal_ compared to _SliceLocalizationBenchmark_
+    These new benchmarks have the same fidelity space as used in previous experiments by [DEHB](https://ml.informatik.uni-freiburg.de/wp-content/uploads/papers/21-IJCAI-DEHB.pdf) and [BOHB](http://proceedings.mlr.press/v80/falkner18a/falkner18a.pdf).
+    Specifically, we increase the lowest fidelity from 1 to 3 for nas/tabular_benchmarks and from 1 to 12 for nas/nasbench_201. The upper fidelity and the old benchmarks remain unchanged.
+
 # 0.0.9
   * Add new Benchmarks: Tabular Benchmarks.
     Provided by @Neeratyoy. 

examples/w_optimizer/SVMSurrogate_minicomparison.py

@@ -0,0 +1,149 @@
+"""
+Multiple Optimizers on SVMSurrogate
+=======================================
+
+This example shows how to run SMAC-HB and SMAC-random-search on SVMSurrogate
+
+Please install the necessary dependencies via ``pip install .`` and singularity (v3.5).
+https://sylabs.io/guides/3.5/user-guide/quick_start.html#quick-installation-steps
+"""
+import logging
+from pathlib import Path
+from time import time
+
+import numpy as np
+from smac.facade.smac_mf_facade import SMAC4MF
+from smac.facade.roar_facade import ROAR
+from smac.intensification.hyperband import Hyperband
+from smac.scenario.scenario import Scenario
+from smac.callbacks import IncorporateRunResultCallback
+
+from hpobench.container.benchmarks.surrogates.svm_benchmark import SurrogateSVMBenchmark
+from hpobench.abstract_benchmark import AbstractBenchmark
+from hpobench.util.example_utils import set_env_variables_to_use_only_one_core
+
+logger = logging.getLogger("minicomp")
+logging.basicConfig(level=logging.INFO)
+set_env_variables_to_use_only_one_core()
+
+
+class Callback(IncorporateRunResultCallback):
+    def __init__(self):
+        self.budget = 10
+
+    def __call__(self, smbo, run_info, result, time_left) -> None:
+        self.budget -= run_info.budget
+        if self.budget < 0:
+            # No budget left
+            raise ValueError
+
+def create_smac_rs(benchmark, output_dir: Path, seed: int):
+    # Set up SMAC-HB
+    cs = benchmark.get_configuration_space(seed=seed)
+
+    scenario_dict = {"run_obj": "quality",  # we optimize quality (alternative to runtime)
+                     "wallclock-limit": 60,
+                     "cs": cs,
+                     "deterministic": "true",
+                     "runcount-limit": 200,
+                     "limit_resources": True,  # Uses pynisher to limit memory and runtime
+                     "cutoff": 1800,  # runtime limit for target algorithm
+                     "memory_limit": 10000,  # adapt this to reasonable value for your hardware
+                     "output_dir": output_dir,
+                     "abort_on_first_run_crash": True,
+                     }
+
+    scenario = Scenario(scenario_dict)
+    def optimization_function_wrapper(cfg, seed, **kwargs):
+        """ Helper-function: simple wrapper to use the benchmark with smac """
+        result_dict = benchmark.objective_function(cfg, rng=seed)
+        cs.sample_configuration()
+        return result_dict['function_value']
+
+    smac = ROAR(scenario=scenario,
+                   rng=np.random.RandomState(seed),
+                   tae_runner=optimization_function_wrapper,
+                   )
+    return smac
+
+def create_smac_hb(benchmark, output_dir: Path, seed: int):
+    # Set up SMAC-HB
+    cs = benchmark.get_configuration_space(seed=seed)
+
+    scenario_dict = {"run_obj": "quality",  # we optimize quality (alternative to runtime)
+                     "wallclock-limit": 60,
+                     "cs": cs,
+                     "deterministic": "true",
+                     "runcount-limit": 200,
+                     "limit_resources": True,  # Uses pynisher to limit memory and runtime
+                     "cutoff": 1800,  # runtime limit for target algorithm
+                     "memory_limit": 10000,  # adapt this to reasonable value for your hardware
+                     "output_dir": output_dir,
+                     "abort_on_first_run_crash": True,
+                     }
+
+    scenario = Scenario(scenario_dict)
+    def optimization_function_wrapper(cfg, seed, instance, budget):
+        """ Helper-function: simple wrapper to use the benchmark with smac """
+        result_dict = benchmark.objective_function(cfg, rng=seed,
+                                                   fidelity={"dataset_fraction": budget})
+        cs.sample_configuration()
+        return result_dict['function_value']
+
+    smac = SMAC4MF(scenario=scenario,
+                   rng=np.random.RandomState(seed),
+                   tae_runner=optimization_function_wrapper,
+                   intensifier=Hyperband,
+                   intensifier_kwargs={'initial_budget': 0.1, 'max_budget': 1, 'eta': 3}
+                   )
+    return smac
+
+
+def run_experiment(out_path: str, on_travis: bool = False):
+
+    out_path = Path(out_path)
+    out_path.mkdir(exist_ok=True)
+
+    hb_res = []
+    rs_res = []
+    for i in range(4):
+        benchmark = SurrogateSVMBenchmark(rng=i)
+        smac = create_smac_hb(benchmark=benchmark, seed=i, output_dir=out_path)
+        callback = Callback()
+        smac.register_callback(callback)
+        try:
+            smac.optimize()
+        except ValueError:
+            print("Done")
+        incumbent = smac.solver.incumbent
+        inc_res = benchmark.objective_function(configuration=incumbent)
+        hb_res.append(inc_res["function_value"])
+
+        benchmark = SurrogateSVMBenchmark(rng=i)
+        smac = create_smac_rs(benchmark=benchmark, seed=i, output_dir=out_path)
+        callback = Callback()
+        smac.register_callback(callback)
+        try:
+            smac.optimize()
+        except ValueError:
+            print("Done")
+        incumbent = smac.solver.incumbent
+        inc_res = benchmark.objective_function(configuration=incumbent)
+        rs_res.append(inc_res["function_value"])
+
+    print("SMAC-HB", hb_res, np.median(hb_res))
+    print("SMAC-RS", rs_res, np.median(rs_res))
+
+
+if __name__ == "__main__":
+    import argparse
+    parser = argparse.ArgumentParser(prog='HPOBench - SVM comp',
+                                     description='Run different opts on SVM Surrogate',
+                                     usage='%(prog)s --out_path <string>')
+    parser.add_argument('--out_path', default='./svm_comp', type=str)
+    parser.add_argument('--on_travis', action='store_true',
+                        help='Flag to speed up the run on the continuous integration tool \"travis\". This flag can be'
+                             'ignored by the user')
+    args = parser.parse_args()
+
+    run_experiment(out_path=args.out_path, on_travis=args.on_travis)

examples/w_optimizer/cartpole_bohb.py

@@ -1,11 +1,13 @@
 """
-SMAC on Cartpole with BOHB
+BOHB on Cartpole
 ==========================
 
 This example shows the usage of an Hyperparameter Tuner, such as BOHB on the cartpole benchmark.
 BOHB is a combination of Bayesian optimization and Hyperband.
 
-Please install the necessary dependencies via ``pip install .`` and singularity (v3.5).
+**Note**: This is a raw benchmark, i.e. it actually runs an algorithms, and will take some time
+
+Please install the necessary dependencies via ``pip install .[examples]`` and singularity (v3.5).
 https://sylabs.io/guides/3.5/user-guide/quick_start.html#quick-installation-steps
 
 """
@@ -48,16 +50,15 @@ def compute(self, config, budget, **kwargs):
 def run_experiment(out_path, on_travis):
 
     settings = {'min_budget': 1,
-                'max_budget': 5,  # Number of Agents, which are trained to solve the cartpole experiment
-                'num_iterations': 10,  # Number of HB brackets
+                'max_budget': 9,  # number of repetitions; this is the fidelity for this bench
+                'num_iterations': 10,  # Set this to a low number for demonstration
                 'eta': 3,
                 'output_dir': Path(out_path)
                 }
     if on_travis:
         settings.update(get_travis_settings('bohb'))
 
-    b = Benchmark(container_source='library://phmueller/automl',
-                  container_name='cartpole')
+    b = Benchmark(rng=1)
 
     b.get_configuration_space(seed=1)
     settings.get('output_dir').mkdir(exist_ok=True)
@@ -105,11 +106,6 @@ def run_experiment(out_path, on_travis):
 
     if not on_travis:
         benchmark = Benchmark(container_source='library://phmueller/automl')
-        # Old API ---- NO LONGER SUPPORTED ---- This will simply ignore the fidelities
-        # incumbent_result = benchmark.objective_function_test(configuration=inc_cfg,
-        #                                                      budget=settings['max_budget'])
-
-        # New API ---- Use this
         incumbent_result = benchmark.objective_function_test(configuration=inc_cfg,
                                                              fidelity={"budget": settings['max_budget']})
         print(incumbent_result)

examples/w_optimizer/cartpole_hyperband.py → examples/w_optimizer/cartpole_smachb.py

@@ -1,26 +1,28 @@
 """
-SMAC on Cartpole with Hyperband
+SMAC-HB on Cartpole with Hyperband
 ===============================
 
 This example shows the usage of an Hyperparameter Tuner, such as SMAC on the cartpole benchmark.
 We use SMAC with Hyperband.
 
-Please install the necessary dependencies via ``pip install .`` and singularity (v3.5).
+**Note**: This is a raw benchmark, i.e. it actually runs an algorithms, and will take some time
+
+Please install the necessary dependencies via ``pip install .[examples]`` and singularity (v3.5).
 https://sylabs.io/guides/3.5/user-guide/quick_start.html#quick-installation-steps
 """
 import logging
 from pathlib import Path
 from time import time
 
 import numpy as np
-from smac.facade.smac_hpo_facade import SMAC4HPO
+from smac.facade.smac_mf_facade import SMAC4MF
 from smac.intensification.hyperband import Hyperband
 from smac.scenario.scenario import Scenario
 
 from hpobench.container.benchmarks.rl.cartpole import CartpoleReduced as Benchmark
 from hpobench.util.example_utils import get_travis_settings, set_env_variables_to_use_only_one_core
 
-logger = logging.getLogger("HB on cartpole")
+logger = logging.getLogger("SMAC-HB on cartpole")
 logging.basicConfig(level=logging.INFO)
 set_env_variables_to_use_only_one_core()
 
@@ -30,15 +32,11 @@ def run_experiment(out_path: str, on_travis: bool = False):
     out_path = Path(out_path)
     out_path.mkdir(exist_ok=True)
 
-    benchmark = Benchmark(container_source='library://phmueller/automl',
-                          container_name='cartpole',
-                          rng=1)
-
-    cs = benchmark.get_configuration_space(seed=1)
+    benchmark = Benchmark(rng=1)
 
-    scenario_dict = {"run_obj": "quality",  # we optimize quality (alternative to runtime)
+    scenario_dict = {"run_obj": "quality",
                      "wallclock-limit": 5 * 60 * 60,  # max duration to run the optimization (in seconds)
-                     "cs": cs,  # configuration space
+                     "cs": benchmark.get_configuration_space(seed=1),
                      "deterministic": "true",
                      "runcount-limit": 200,
                      "limit_resources": True,  # Uses pynisher to limit memory and runtime
@@ -53,16 +51,14 @@ def run_experiment(out_path: str, on_travis: bool = False):
     scenario = Scenario(scenario_dict)
 
     # Number of Agents, which are trained to solve the cartpole experiment
-    max_budget = 5 if not on_travis else 2
+    max_budget = 9 if not on_travis else 2
 
     def optimization_function_wrapper(cfg, seed, instance, budget):
         """ Helper-function: simple wrapper to use the benchmark with smac"""
 
-        # Now that we have already downloaded the conatiner,
+        # Now that we have already downloaded the container,
         # we only have to start a new instance. This is a fast operation.
-        b = Benchmark(container_source='library://phmueller/automl',
-                      container_name='cartpole',
-                      rng=seed)
+        b = Benchmark(rng=seed)
 
         # Old API ---- NO LONGER SUPPORTED ---- This will simply ignore the fidelities
         # result_dict = b.objective_function(cfg, budget=int(budget))
@@ -71,10 +67,10 @@ def optimization_function_wrapper(cfg, seed, instance, budget):
         result_dict = b.objective_function(cfg, fidelity={"budget": int(budget)})
         return result_dict['function_value']
 
-    smac = SMAC4HPO(scenario=scenario,
+    smac = SMAC4MF(scenario=scenario,
                     rng=np.random.RandomState(42),
                     tae_runner=optimization_function_wrapper,
-                    intensifier=Hyperband,  # you can also change the intensifier to use like this!
+                    intensifier=Hyperband,
                     intensifier_kwargs={'initial_budget': 1, 'max_budget': max_budget, 'eta': 3}
                     )