v0.1.1 (#154)

* fix: typo * build: update scripts * ignore python * docs: update install instructions * docs: update install instructions * build: fix makefile typo * docs: fix typos * build: updates for hpobench installation * build: updates for hpobench installation * Add logo to README.md + docs; adapt docs theme colors to logo * Actually add the logo file * fix: changelog entry for initial version * add running data to readme (#139) * add running data to readme * Update README.md --------- Co-authored-by: benjamc <[email protected]> * Update plotting * Update filepath * fix(check_missing.py): Fix criterion for max trials (#150) For MF, max trials can be a mu above integer value specified because of the partial trials. Fix the condition. * Fix HEBO integration (#149) * fix(hebo): Do not increase trial_counter twice Must have been forgotton 😭💀 * Update CHANGELOG.md * Update CHANGELOG.md * Update version number 0.1.0->0.1.1 * Feat/plotting (#152) * refactor(plotting): rm old notebooks * refactor: update plotting of subselections * Fix/dist (#153) * Remove dependencies * fix(plotting): small fixes * Update CHANGELOG.md --------- Co-authored-by: Sarah Krebs <[email protected]> Co-authored-by: Helena Graf <[email protected]> Co-authored-by: Sarah Segel <[email protected]> Co-authored-by: Difan Deng <[email protected]>
automl · Jul 1, 2024 · 7c3d2ad · 7c3d2ad
1 parent f5f8b95
commit 7c3d2ad
Show file tree

Hide file tree

Showing 12 changed files with 2,548 additions and 2,425 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,6 +3,7 @@
 
 ## Bug Fixes
 - Fix HEBO integration (#149)
+- Fix plotting and data gathering
 
 # v0.1.0
 

diff --git a/carps/analysis/concat_rundata.py b/carps/analysis/concat_rundata.py
@@ -0,0 +1,36 @@
+from __future__ import annotations
+import pandas as pd
+from carps.analysis.gather_data import load_set, convert_mixed_types_to_str
+
+def concat_rundata():
+    paths = {
+        "BBsubset": {
+            "dev": ["runs_subset_BB/dev"],
+            "test": ["runs_subset_BB/test"],
+        },
+        "MFsubset": {
+            "dev": ["runs_subset_MF/dev"],
+            "test": ["runs_subset_MF/test"],
+        },
+        "MOsubset": {
+            "dev": ["runs_subset_MO/dev"],
+            "test": ["runs_subset_MO/test"],
+        },
+        "MOMFsubset": {
+            "dev": ["runs_subset_MOMF/dev"],
+            "test": ["runs_subset_MOMF/test"],
+        },
+    }
+
+    args = []
+    for item in paths.values():
+        for k,v in item.items():
+            from pathlib import Path
+            args.append((v,k))
+    res = [load_set(paths=a[0], set_id=a[1]) for a in args]
+    df = pd.concat([r[0] for r in res]).reset_index(drop=True)
+    df = convert_mixed_types_to_str(df)
+    df.to_parquet("rundata.parquet")
+
+if __name__ == "__main__":
+    concat_rundata()
diff --git a/carps/analysis/interval_estimates.py b/carps/analysis/interval_estimates.py
diff --git a/carps/analysis/performance_over_time.py b/carps/analysis/performance_over_time.py
@@ -14,8 +14,8 @@
 if TYPE_CHECKING:
     import pandas as pd
 
-def get_order_by_mean(df: pd.DataFrame) -> list[str]:
-    final_df = filter_only_final_performance(df)
+def get_order_by_mean(df: pd.DataFrame, budget_var: str = "n_trials_norm") -> list[str]:
+    final_df = filter_only_final_performance(df, budget_var=budget_var)
     reduced = final_df.groupby(by="optimizer_id")["trial_value__cost_inc_norm"].apply(np.nanmean)
     reduced = reduced.sort_values()
     return reduced.index.tolist()
@@ -33,7 +33,7 @@ def plot_performance_over_time(
     **lineplot_kwargs,
 ) -> tuple[plt.Figure, matplotlib.axes.Axes]:
     setup_seaborn(font_scale=1.5)
-    sorter = get_order_by_mean(df=df)
+    sorter = get_order_by_mean(df=df, budget_var=x)
     df = df.sort_values(by="optimizer_id", key=lambda column: column.map(lambda e: sorter.index(e)))
     palette = get_color_palette(df)
     fig = plt.figure(figsize=figsize)

diff --git a/notebooks/calc_runtime.py b/notebooks/calc_runtime.py
@@ -1,39 +1,10 @@
 from __future__ import annotations
 
-from carps.analysis.gather_data import load_set, convert_mixed_types_to_str
 from carps.analysis.utils import filter_only_final_performance
 import pandas as pd
 import multiprocessing
 
-paths = {
-    "BBsubset": {
-        "dev": ["../runs_subset_BB/dev"],
-        "test": ["../runs_subset_BB/test"],
-    },
-    "MFsubset": {
-        "dev": ["../runs_subset_MF/dev"],
-        "test": ["../runs_subset_MF/test"],
-    },
-    "MOsubset": {
-        "dev": ["../runs_subset_MO/dev"],
-        "test": ["../runs_subset_MO/test"],
-    },
-    "MOMFsubset": {
-        "dev": ["../runs_subset_MOMF/dev"],
-        "test": ["../runs_subset_MOMF/test"],
-    },
-}
-
-
-
-# args = []
-# for item in paths.values():
-#     for k,v in item.items():
-#         args.append((v,k))
-# res = [load_set(paths=a[0], set_id=a[1]) for a in args]
-# df = pd.concat([r[0] for r in res]).reset_index(drop=True)
-# df = convert_mixed_types_to_str(df)
-# df.to_parquet("rundata.parquet")
+concat_rundata()
 
 df  = pd.read_parquet("rundata.parquet")
 df_final = filter_only_final_performance(df=df, budget_var="n_trials")