Sarah Segel: Merge pull request #188 from automl/feature/mo_hpi

development/.buildinfo

@@ -1,4 +1,4 @@
 # Sphinx build info version 1
 # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
-config: 764570ae7810d5753a967b6a264633d8
+config: 5f5cc67e9884aa53630cf60444ed0069
 tags: 645f666f9bcd5a90fca523b33c5a78b7

development/_downloads/aabae18a223fa014a3ff3692699ee5e2/importances.ipynb

@@ -15,7 +15,7 @@
       },
       "outputs": [],
       "source": [
-        "from deepcave.plugins.hyperparameter.importances import Importances\nfrom deepcave.runs.converters.deepcave import DeepCAVERun\nfrom pathlib import Path\n\n\nif __name__ == \"__main__\":\n    # Instantiate the run\n    run = DeepCAVERun.from_path(Path(\"logs/DeepCAVE/minimal/run_2\"))\n\n    objective_id = run.get_objective_ids()[0]\n    budget_ids = run.get_budget_ids()\n\n    # Instantiate the plugin\n    plugin = Importances()\n    inputs = plugin.generate_inputs(\n        hyperparameter_names=list(run.configspace.keys()),\n        objective_id=objective_id,\n        budget_ids=budget_ids,\n        method=\"global\",\n        n_hps=3,\n        n_trees=10,\n    )\n    # Note: Filter variables are not considered.\n    outputs = plugin.generate_outputs(run, inputs)\n\n    # Finally, you can load the figure. Here, the filter variables play a role.\n    figure = plugin.load_outputs(run, inputs, outputs)\n    figure.write_image(\"examples/api/importances.png\", scale=2.0)"
+        "from deepcave.plugins.hyperparameter.importances import Importances\nfrom deepcave.runs.converters.deepcave import DeepCAVERun\nfrom pathlib import Path\n\n\nif __name__ == \"__main__\":\n    # Instantiate the run\n    run = DeepCAVERun.from_path(Path(\"logs/DeepCAVE/minimal/run_2\"))\n\n    objective_id1 = run.get_objective_ids()[0]\n    objective_id2 = None  # replace with run.get_objective_ids()[1] for multi-objective importance\n    budget_ids = run.get_budget_ids()\n\n    # Instantiate the plugin\n    plugin = Importances()\n    inputs = plugin.generate_inputs(\n        hyperparameter_names=list(run.configspace.keys()),\n        objective_id1=objective_id1,\n        objective_id2=objective_id2,\n        budget_ids=budget_ids,\n        method=\"global\",\n        n_hps=3,\n        n_trees=10,\n    )\n    # Note: Filter variables are not considered.\n    outputs = plugin.generate_outputs(run, inputs)\n\n    # Finally, you can load the figure. Here, the filter variables play a role.\n    figure = plugin.load_outputs(run, inputs, outputs)\n    figure.write_image(\"examples/api/importances.png\", scale=2.0)"
       ]
     }
   ],

development/_downloads/d1732d869fba86e022de98932899c009/ablation_paths.ipynb

@@ -15,7 +15,7 @@
       },
       "outputs": [],
       "source": [
-        "from deepcave.plugins.hyperparameter.ablation_paths import AblationPaths\nfrom deepcave.runs.converters.deepcave import DeepCAVERun\nfrom pathlib import Path\n\n\nif __name__ == \"__main__\":\n    # Instantiate the run\n    run = DeepCAVERun.from_path(Path(\"logs/DeepCAVE/minimal/run_2\"))\n\n    objective_id = run.get_objective_ids()[0]\n    budget_id = run.get_budget_ids()[0]\n\n    # Instantiate the plugin\n    plugin = AblationPaths()\n    inputs = plugin.generate_inputs(\n        objective_id=objective_id,\n        budget_id=budget_id,\n        n_hps=100,\n        n_trees=100,\n        show_confidence=False,\n    )\n    # Note: Filter variables are not considered.\n    outputs = plugin.generate_outputs(run, inputs)\n\n    # Finally, you can load the figure. Here, the filter variables play a role.\n    figure1, figure2 = plugin.load_outputs(run, inputs, outputs)\n    figure1.write_image(\"examples/api/ablation_paths_performance.png\", scale=2.0)\n    figure2.write_image(\"examples/api/ablation_paths_improvement.png\", scale=2.0)"
+        "from deepcave.plugins.hyperparameter.ablation_paths import AblationPaths\nfrom deepcave.runs.converters.deepcave import DeepCAVERun\nfrom pathlib import Path\n\n\nif __name__ == \"__main__\":\n    # Instantiate the run\n    run = DeepCAVERun.from_path(Path(\"logs/DeepCAVE/minimal/run_1\"))\n\n    objective_id1 = run.get_objective_ids()[0]\n    objective_id2 = None  # replace with run.get_objective_ids()[1] for multi-objective importance\n    budget_id = run.get_budget_ids()[0]\n\n    # Instantiate the plugin\n    plugin = AblationPaths()\n    inputs = plugin.generate_inputs(\n        objective_id1=objective_id1,\n        objective_id2=objective_id2,\n        budget_id=budget_id,\n        n_hps=100,\n        n_trees=100,\n        show_confidence=False,\n    )\n    # Note: Filter variables are not considered.\n    outputs = plugin.generate_outputs(run, inputs)\n\n    # Finally, you can load the figure. Here, the filter variables play a role.\n    figure1, figure2 = plugin.load_outputs(run, inputs, outputs)\n    figure1.write_image(\"examples/api/ablation_paths_performance.png\", scale=2.0)\n    if not objective_id2:\n        figure2.write_image(\"examples/api/ablation_paths_improvement.png\", scale=2.0)"
       ]
     }
   ],

development/_downloads/e982a81887130c71eb62b90f17e126fa/ablation_paths.py

@@ -13,15 +13,17 @@
 
 if __name__ == "__main__":
     # Instantiate the run
-    run = DeepCAVERun.from_path(Path("logs/DeepCAVE/minimal/run_2"))
+    run = DeepCAVERun.from_path(Path("logs/DeepCAVE/minimal/run_1"))
 
-    objective_id = run.get_objective_ids()[0]
+    objective_id1 = run.get_objective_ids()[0]
+    objective_id2 = None  # replace with run.get_objective_ids()[1] for multi-objective importance
     budget_id = run.get_budget_ids()[0]
 
     # Instantiate the plugin
     plugin = AblationPaths()
     inputs = plugin.generate_inputs(
-        objective_id=objective_id,
+        objective_id1=objective_id1,
+        objective_id2=objective_id2,
         budget_id=budget_id,
         n_hps=100,
         n_trees=100,
@@ -33,4 +35,5 @@
     # Finally, you can load the figure. Here, the filter variables play a role.
     figure1, figure2 = plugin.load_outputs(run, inputs, outputs)
     figure1.write_image("examples/api/ablation_paths_performance.png", scale=2.0)
-    figure2.write_image("examples/api/ablation_paths_improvement.png", scale=2.0)
+    if not objective_id2:
+        figure2.write_image("examples/api/ablation_paths_improvement.png", scale=2.0)

development/_downloads/f0660e643860808c88e59e2b47bd432b/importances.py

@@ -15,14 +15,16 @@
     # Instantiate the run
     run = DeepCAVERun.from_path(Path("logs/DeepCAVE/minimal/run_2"))
 
-    objective_id = run.get_objective_ids()[0]
+    objective_id1 = run.get_objective_ids()[0]
+    objective_id2 = None  # replace with run.get_objective_ids()[1] for multi-objective importance
     budget_ids = run.get_budget_ids()
 
     # Instantiate the plugin
     plugin = Importances()
     inputs = plugin.generate_inputs(
         hyperparameter_names=list(run.configspace.keys()),
-        objective_id=objective_id,
+        objective_id1=objective_id1,
+        objective_id2=objective_id2,
         budget_ids=budget_ids,
         method="global",
         n_hps=3,

development/_modules/deepcave/evaluators/ablation.html

@@ -4,7 +4,7 @@
   <head>
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>deepcave.evaluators.ablation &#8212; DeepCAVE Documentation 1.3.1 documentation</title>
+    <title>deepcave.evaluators.ablation &#8212; DeepCAVE Documentation 1.3.2 documentation</title>
 
   <link href="../../../_static/css/theme.css" rel="stylesheet">
   <link href="../../../_static/css/index.ac9c05f7c49ca1e1f876c6e36360ea26.css" rel="stylesheet">
@@ -30,7 +30,7 @@
 
   <link rel="preload" as="script" href="../../../_static/js/index.9ea38e314b9e6d9dab77.js">
 
-    <script src="../../../_static/documentation_options.js?v=4aaf858c"></script>
+    <script src="../../../_static/documentation_options.js?v=6a1e1f67"></script>
     <script src="../../../_static/doctools.js?v=9a2dae69"></script>
     <script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
     <link rel="icon" href="../../../_static/favicon.ico"/>
@@ -93,11 +93,11 @@
         <div class="dropdown mt-1">
     <button type="button" class="btn btn-secondary btn-md navbar-btn dropdown-toggle" id="dLabelMore"
         data-toggle="dropdown">
-        v1.3.1
+        v1.3.2
         <span class="caret"></span>
     </button>
     <div class="dropdown-menu list-group-flush py-0" aria-labelledby="dLabelMore">
-        <a class="list-group-item list-group-item-action" href="#">v1.3.1</a>
+        <a class="list-group-item list-group-item-action" href="#">v1.3.2</a>
     </div>
 </div>
       </div>
@@ -122,7 +122,7 @@
 
             <!-- Only show if we have sidebars configured, else just a small margin  -->
             <div class="col-12 col-md-3 bd-sidebar"><h4 class="mt-0 mb-0"><a href="../../../index.html">DeepCAVE Documentation</a></h4>
-<div class="mb-3">v1.3.1</div><nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
+<div class="mb-3">v1.3.2</div><nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
   <div class="bd-toc-item active">
     <ul class="nav bd-sidenav">
  <li class="toctree-l1">
@@ -160,6 +160,11 @@
        Partial Dependencies
       </a>
      </li>
+     <li class="toctree-l3">
+      <a class="reference internal" href="../../../examples/api/parallel_coordinates.html">
+       Parallel Coordinates
+      </a>
+     </li>
      <li class="toctree-l3">
       <a class="reference internal" href="../../../examples/api/importances.html">
        Importances
@@ -170,11 +175,6 @@
        Ablation Paths
       </a>
      </li>
-     <li class="toctree-l3">
-      <a class="reference internal" href="../../../examples/api/parallel_coordinates.html">
-       Parallel Coordinates
-      </a>
-     </li>
     </ul>
    </li>
    <li class="toctree-l2 has-children">
@@ -634,6 +634,21 @@
        deepcave.evaluators.lpi
       </a>
      </li>
+     <li class="toctree-l3">
+      <a class="reference internal" href="../../../api/deepcave.evaluators.mo_ablation.html">
+       deepcave.evaluators.mo_ablation
+      </a>
+     </li>
+     <li class="toctree-l3">
+      <a class="reference internal" href="../../../api/deepcave.evaluators.mo_fanova.html">
+       deepcave.evaluators.mo_fanova
+      </a>
+     </li>
+     <li class="toctree-l3">
+      <a class="reference internal" href="../../../api/deepcave.evaluators.mo_lpi.html">
+       deepcave.evaluators.mo_lpi
+      </a>
+     </li>
     </ul>
    </li>
    <li class="toctree-l2 has-children">
@@ -711,6 +726,11 @@
        deepcave.utils.logs
       </a>
      </li>
+     <li class="toctree-l3">
+      <a class="reference internal" href="../../../api/deepcave.utils.multi_objective_importance.html">
+       deepcave.utils.multi_objective_importance
+      </a>
+     </li>
      <li class="toctree-l3">
       <a class="reference internal" href="../../../api/deepcave.utils.notification.html">
        deepcave.utils.notification
@@ -1036,7 +1056,7 @@ <h1>Source code for deepcave.evaluators.ablation</h1><div class="highlight"><pre
         <span class="n">max_hp_difference</span> <span class="o">=</span> <span class="o">-</span><span class="n">np</span><span class="o">.</span><span class="n">inf</span>
 
         <span class="k">for</span> <span class="n">hp</span> <span class="ow">in</span> <span class="n">hp_it</span><span class="p">:</span>
-            <span class="k">if</span> <span class="n">hp</span> <span class="ow">in</span> <span class="n">hp</span> <span class="ow">in</span> <span class="n">incumbent_config</span><span class="o">.</span><span class="n">keys</span><span class="p">()</span> <span class="ow">and</span> <span class="n">hp</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">default_config</span><span class="o">.</span><span class="n">keys</span><span class="p">():</span>
+            <span class="k">if</span> <span class="n">hp</span> <span class="ow">in</span> <span class="n">incumbent_config</span><span class="o">.</span><span class="n">keys</span><span class="p">()</span> <span class="ow">and</span> <span class="n">hp</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">default_config</span><span class="o">.</span><span class="n">keys</span><span class="p">():</span>
                 <span class="n">config_copy</span> <span class="o">=</span> <span class="n">copy</span><span class="o">.</span><span class="n">copy</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">default_config</span><span class="p">)</span>
                 <span class="n">config_copy</span><span class="p">[</span><span class="n">hp</span><span class="p">]</span> <span class="o">=</span> <span class="n">incumbent_config</span><span class="p">[</span><span class="n">hp</span><span class="p">]</span>
 

development/_modules/deepcave/evaluators/epm/fanova_forest.html

@@ -4,7 +4,7 @@
   <head>
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>deepcave.evaluators.epm.fanova_forest &#8212; DeepCAVE Documentation 1.3.1 documentation</title>
+    <title>deepcave.evaluators.epm.fanova_forest &#8212; DeepCAVE Documentation 1.3.2 documentation</title>
 
   <link href="../../../../_static/css/theme.css" rel="stylesheet">
   <link href="../../../../_static/css/index.ac9c05f7c49ca1e1f876c6e36360ea26.css" rel="stylesheet">
@@ -30,7 +30,7 @@
 
   <link rel="preload" as="script" href="../../../../_static/js/index.9ea38e314b9e6d9dab77.js">
 
-    <script src="../../../../_static/documentation_options.js?v=4aaf858c"></script>
+    <script src="../../../../_static/documentation_options.js?v=6a1e1f67"></script>
     <script src="../../../../_static/doctools.js?v=9a2dae69"></script>
     <script src="../../../../_static/sphinx_highlight.js?v=dc90522c"></script>
     <link rel="icon" href="../../../../_static/favicon.ico"/>
@@ -93,11 +93,11 @@
         <div class="dropdown mt-1">
     <button type="button" class="btn btn-secondary btn-md navbar-btn dropdown-toggle" id="dLabelMore"
         data-toggle="dropdown">
-        v1.3.1
+        v1.3.2
         <span class="caret"></span>
     </button>
     <div class="dropdown-menu list-group-flush py-0" aria-labelledby="dLabelMore">
-        <a class="list-group-item list-group-item-action" href="#">v1.3.1</a>
+        <a class="list-group-item list-group-item-action" href="#">v1.3.2</a>
     </div>
 </div>
       </div>
@@ -122,7 +122,7 @@
 
             <!-- Only show if we have sidebars configured, else just a small margin  -->
             <div class="col-12 col-md-3 bd-sidebar"><h4 class="mt-0 mb-0"><a href="../../../../index.html">DeepCAVE Documentation</a></h4>
-<div class="mb-3">v1.3.1</div><nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
+<div class="mb-3">v1.3.2</div><nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
   <div class="bd-toc-item active">
     <ul class="nav bd-sidenav">
  <li class="toctree-l1">
@@ -160,6 +160,11 @@
        Partial Dependencies
       </a>
      </li>
+     <li class="toctree-l3">
+      <a class="reference internal" href="../../../../examples/api/parallel_coordinates.html">
+       Parallel Coordinates
+      </a>
+     </li>
      <li class="toctree-l3">
       <a class="reference internal" href="../../../../examples/api/importances.html">
        Importances
@@ -170,11 +175,6 @@
        Ablation Paths
       </a>
      </li>
-     <li class="toctree-l3">
-      <a class="reference internal" href="../../../../examples/api/parallel_coordinates.html">
-       Parallel Coordinates
-      </a>
-     </li>
     </ul>
    </li>
    <li class="toctree-l2 has-children">
@@ -634,6 +634,21 @@
        deepcave.evaluators.lpi
       </a>
      </li>
+     <li class="toctree-l3">
+      <a class="reference internal" href="../../../../api/deepcave.evaluators.mo_ablation.html">
+       deepcave.evaluators.mo_ablation
+      </a>
+     </li>
+     <li class="toctree-l3">
+      <a class="reference internal" href="../../../../api/deepcave.evaluators.mo_fanova.html">
+       deepcave.evaluators.mo_fanova
+      </a>
+     </li>
+     <li class="toctree-l3">
+      <a class="reference internal" href="../../../../api/deepcave.evaluators.mo_lpi.html">
+       deepcave.evaluators.mo_lpi
+      </a>
+     </li>
     </ul>
    </li>
    <li class="toctree-l2 has-children">
@@ -711,6 +726,11 @@
        deepcave.utils.logs
       </a>
      </li>
+     <li class="toctree-l3">
+      <a class="reference internal" href="../../../../api/deepcave.utils.multi_objective_importance.html">
+       deepcave.utils.multi_objective_importance
+      </a>
+     </li>
      <li class="toctree-l3">
       <a class="reference internal" href="../../../../api/deepcave.utils.notification.html">
        deepcave.utils.notification