From a1fba892c7aec8af07a2b6c49c97f671dde0a09b Mon Sep 17 00:00:00 2001 From: Carolin Benjamins Date: Tue, 4 Jun 2024 16:47:09 +0200 Subject: [PATCH] feat(subselection): add subset for MO --- .../dev/subset_Pymoo_MO_omnitest.yaml | 35 ++++++++++++++++++ .../subset_hpobench_MO_tab_ml_rf_167119.yaml | 37 +++++++++++++++++++ .../dev/subset_hpobench_MO_tab_ml_rf_31.yaml | 37 +++++++++++++++++++ .../subset_hpobench_MO_tab_ml_svm_146212.yaml | 37 +++++++++++++++++++ .../subset_hpobench_MO_tab_ml_svm_146818.yaml | 37 +++++++++++++++++++ .../dev/subset_hpobench_MO_tab_ml_svm_31.yaml | 37 +++++++++++++++++++ ...set_hpobench_MO_tab_ml_xgboost_146822.yaml | 37 +++++++++++++++++++ ...bset_hpobench_MO_tab_ml_xgboost_14965.yaml | 37 +++++++++++++++++++ ...set_hpobench_MO_tab_ml_xgboost_167120.yaml | 37 +++++++++++++++++++ ...subset_yahpo_mo_rbv2_rpart_40499_None.yaml | 36 ++++++++++++++++++ .../test/subset_Pymoo_MO_zdt3.yaml | 35 ++++++++++++++++++ .../subset_hpobench_MO_tab_ml_lr_146606.yaml | 37 +++++++++++++++++++ .../subset_hpobench_MO_tab_ml_lr_167119.yaml | 37 +++++++++++++++++++ .../subset_hpobench_MO_tab_ml_lr_168912.yaml | 37 +++++++++++++++++++ .../subset_hpobench_MO_tab_ml_nn_146821.yaml | 37 +++++++++++++++++++ .../subset_hpobench_MO_tab_ml_rf_146212.yaml | 37 +++++++++++++++++++ .../subset_hpobench_MO_tab_ml_svm_10101.yaml | 37 +++++++++++++++++++ .../subset_hpobench_MO_tab_ml_svm_146821.yaml | 37 +++++++++++++++++++ .../subset_hpobench_MO_tab_ml_svm_7592.yaml | 37 +++++++++++++++++++ ...set_hpobench_MO_tab_ml_xgboost_168911.yaml | 37 +++++++++++++++++++ .../dev/subset_Pymoo_MO_omnitest.yaml | 35 ++++++++++++++++++ .../subset_hpobench_MO_tab_ml_rf_167119.yaml | 37 +++++++++++++++++++ .../dev/subset_hpobench_MO_tab_ml_rf_31.yaml | 37 +++++++++++++++++++ .../subset_hpobench_MO_tab_ml_svm_146212.yaml | 37 +++++++++++++++++++ .../subset_hpobench_MO_tab_ml_svm_146818.yaml | 37 +++++++++++++++++++ .../dev/subset_hpobench_MO_tab_ml_svm_31.yaml | 37 +++++++++++++++++++ ...set_hpobench_MO_tab_ml_xgboost_146822.yaml | 37 +++++++++++++++++++ ...bset_hpobench_MO_tab_ml_xgboost_14965.yaml | 37 +++++++++++++++++++ ...set_hpobench_MO_tab_ml_xgboost_167120.yaml | 37 +++++++++++++++++++ ...subset_yahpo_mo_rbv2_rpart_40499_None.yaml | 36 ++++++++++++++++++ .../test/subset_Pymoo_MO_zdt3.yaml | 35 ++++++++++++++++++ .../subset_hpobench_MO_tab_ml_lr_146606.yaml | 37 +++++++++++++++++++ .../subset_hpobench_MO_tab_ml_lr_167119.yaml | 37 +++++++++++++++++++ .../subset_hpobench_MO_tab_ml_lr_168912.yaml | 37 +++++++++++++++++++ .../subset_hpobench_MO_tab_ml_nn_146821.yaml | 37 +++++++++++++++++++ .../subset_hpobench_MO_tab_ml_rf_146212.yaml | 37 +++++++++++++++++++ .../subset_hpobench_MO_tab_ml_svm_10101.yaml | 37 +++++++++++++++++++ .../subset_hpobench_MO_tab_ml_svm_146821.yaml | 37 +++++++++++++++++++ .../subset_hpobench_MO_tab_ml_svm_7592.yaml | 37 +++++++++++++++++++ ...set_hpobench_MO_tab_ml_xgboost_168911.yaml | 37 +++++++++++++++++++ subselection/create_subset_configs.py | 4 +- 41 files changed, 1473 insertions(+), 1 deletion(-) create mode 100644 carps/configs/problem/subselection/multifidelity/dev/subset_Pymoo_MO_omnitest.yaml create mode 100644 carps/configs/problem/subselection/multifidelity/dev/subset_hpobench_MO_tab_ml_rf_167119.yaml create mode 100644 carps/configs/problem/subselection/multifidelity/dev/subset_hpobench_MO_tab_ml_rf_31.yaml create mode 100644 carps/configs/problem/subselection/multifidelity/dev/subset_hpobench_MO_tab_ml_svm_146212.yaml create mode 100644 carps/configs/problem/subselection/multifidelity/dev/subset_hpobench_MO_tab_ml_svm_146818.yaml create mode 100644 carps/configs/problem/subselection/multifidelity/dev/subset_hpobench_MO_tab_ml_svm_31.yaml create mode 100644 carps/configs/problem/subselection/multifidelity/dev/subset_hpobench_MO_tab_ml_xgboost_146822.yaml create mode 100644 carps/configs/problem/subselection/multifidelity/dev/subset_hpobench_MO_tab_ml_xgboost_14965.yaml create mode 100644 carps/configs/problem/subselection/multifidelity/dev/subset_hpobench_MO_tab_ml_xgboost_167120.yaml create mode 100644 carps/configs/problem/subselection/multifidelity/dev/subset_yahpo_mo_rbv2_rpart_40499_None.yaml create mode 100644 carps/configs/problem/subselection/multifidelity/test/subset_Pymoo_MO_zdt3.yaml create mode 100644 carps/configs/problem/subselection/multifidelity/test/subset_hpobench_MO_tab_ml_lr_146606.yaml create mode 100644 carps/configs/problem/subselection/multifidelity/test/subset_hpobench_MO_tab_ml_lr_167119.yaml create mode 100644 carps/configs/problem/subselection/multifidelity/test/subset_hpobench_MO_tab_ml_lr_168912.yaml create mode 100644 carps/configs/problem/subselection/multifidelity/test/subset_hpobench_MO_tab_ml_nn_146821.yaml create mode 100644 carps/configs/problem/subselection/multifidelity/test/subset_hpobench_MO_tab_ml_rf_146212.yaml create mode 100644 carps/configs/problem/subselection/multifidelity/test/subset_hpobench_MO_tab_ml_svm_10101.yaml create mode 100644 carps/configs/problem/subselection/multifidelity/test/subset_hpobench_MO_tab_ml_svm_146821.yaml create mode 100644 carps/configs/problem/subselection/multifidelity/test/subset_hpobench_MO_tab_ml_svm_7592.yaml create mode 100644 carps/configs/problem/subselection/multifidelity/test/subset_hpobench_MO_tab_ml_xgboost_168911.yaml create mode 100644 carps/configs/problem/subselection/multiobjective/dev/subset_Pymoo_MO_omnitest.yaml create mode 100644 carps/configs/problem/subselection/multiobjective/dev/subset_hpobench_MO_tab_ml_rf_167119.yaml create mode 100644 carps/configs/problem/subselection/multiobjective/dev/subset_hpobench_MO_tab_ml_rf_31.yaml create mode 100644 carps/configs/problem/subselection/multiobjective/dev/subset_hpobench_MO_tab_ml_svm_146212.yaml create mode 100644 carps/configs/problem/subselection/multiobjective/dev/subset_hpobench_MO_tab_ml_svm_146818.yaml create mode 100644 carps/configs/problem/subselection/multiobjective/dev/subset_hpobench_MO_tab_ml_svm_31.yaml create mode 100644 carps/configs/problem/subselection/multiobjective/dev/subset_hpobench_MO_tab_ml_xgboost_146822.yaml create mode 100644 carps/configs/problem/subselection/multiobjective/dev/subset_hpobench_MO_tab_ml_xgboost_14965.yaml create mode 100644 carps/configs/problem/subselection/multiobjective/dev/subset_hpobench_MO_tab_ml_xgboost_167120.yaml create mode 100644 carps/configs/problem/subselection/multiobjective/dev/subset_yahpo_mo_rbv2_rpart_40499_None.yaml create mode 100644 carps/configs/problem/subselection/multiobjective/test/subset_Pymoo_MO_zdt3.yaml create mode 100644 carps/configs/problem/subselection/multiobjective/test/subset_hpobench_MO_tab_ml_lr_146606.yaml create mode 100644 carps/configs/problem/subselection/multiobjective/test/subset_hpobench_MO_tab_ml_lr_167119.yaml create mode 100644 carps/configs/problem/subselection/multiobjective/test/subset_hpobench_MO_tab_ml_lr_168912.yaml create mode 100644 carps/configs/problem/subselection/multiobjective/test/subset_hpobench_MO_tab_ml_nn_146821.yaml create mode 100644 carps/configs/problem/subselection/multiobjective/test/subset_hpobench_MO_tab_ml_rf_146212.yaml create mode 100644 carps/configs/problem/subselection/multiobjective/test/subset_hpobench_MO_tab_ml_svm_10101.yaml create mode 100644 carps/configs/problem/subselection/multiobjective/test/subset_hpobench_MO_tab_ml_svm_146821.yaml create mode 100644 carps/configs/problem/subselection/multiobjective/test/subset_hpobench_MO_tab_ml_svm_7592.yaml create mode 100644 carps/configs/problem/subselection/multiobjective/test/subset_hpobench_MO_tab_ml_xgboost_168911.yaml diff --git a/carps/configs/problem/subselection/multifidelity/dev/subset_Pymoo_MO_omnitest.yaml b/carps/configs/problem/subselection/multifidelity/dev/subset_Pymoo_MO_omnitest.yaml new file mode 100644 index 00000000..70ee8472 --- /dev/null +++ b/carps/configs/problem/subselection/multifidelity/dev/subset_Pymoo_MO_omnitest.yaml @@ -0,0 +1,35 @@ +# @package _global_ +benchmark_id: Pymoo +problem_id: multifidelity/10/dev/Pymoo/MO/omnitest +problem: + _target_: carps.benchmarks.pymoo.PymooProblem + problem_name: omnitest + seed: ${seed} + metric: + - objective_0 + - objective_1 +task: + n_trials: 77 + time_budget: null + n_workers: 1 + n_objectives: 2 + objectives: + - objective_0 + - objective_1 + is_multifidelity: false + fidelity_type: null + min_budget: null + max_budget: null + has_constraints: false + domain: synthetic + objective_function_approximation: real + has_virtual_time: false + deterministic: true + dimensions: 2 + search_space_n_categoricals: 0 + search_space_n_ordinals: 0 + search_space_n_integers: 0 + search_space_n_floats: 2 + search_space_has_conditionals: false + search_space_has_forbiddens: false + search_space_has_priors: false diff --git a/carps/configs/problem/subselection/multifidelity/dev/subset_hpobench_MO_tab_ml_rf_167119.yaml b/carps/configs/problem/subselection/multifidelity/dev/subset_hpobench_MO_tab_ml_rf_167119.yaml new file mode 100644 index 00000000..3fc6dfe8 --- /dev/null +++ b/carps/configs/problem/subselection/multifidelity/dev/subset_hpobench_MO_tab_ml_rf_167119.yaml @@ -0,0 +1,37 @@ +# @package _global_ +benchmark_id: HPOBench +problem_id: multifidelity/10/dev/hpobench/MO/tab/ml/rf/167119 +problem: + _target_: carps.benchmarks.hpo_bench.HPOBenchProblem + model: rf + task_id: 167119 + metric: + - function_value + - cost + budget_type: null + seed: ${seed} +task: + n_trials: 100 + time_budget: null + n_workers: 1 + n_objectives: 2 + objectives: + - function_value + - cost + is_multifidelity: false + fidelity_type: null + min_budget: null + max_budget: null + has_constraints: false + domain: ML + objective_function_approximation: tabular + has_virtual_time: false + deterministic: false + dimensions: 4 + search_space_n_categoricals: 0 + search_space_n_ordinals: 4 + search_space_n_integers: 0 + search_space_n_floats: 0 + search_space_has_conditionals: false + search_space_has_forbiddens: false + search_space_has_priors: false diff --git a/carps/configs/problem/subselection/multifidelity/dev/subset_hpobench_MO_tab_ml_rf_31.yaml b/carps/configs/problem/subselection/multifidelity/dev/subset_hpobench_MO_tab_ml_rf_31.yaml new file mode 100644 index 00000000..0ae7606d --- /dev/null +++ b/carps/configs/problem/subselection/multifidelity/dev/subset_hpobench_MO_tab_ml_rf_31.yaml @@ -0,0 +1,37 @@ +# @package _global_ +benchmark_id: HPOBench +problem_id: multifidelity/10/dev/hpobench/MO/tab/ml/rf/31 +problem: + _target_: carps.benchmarks.hpo_bench.HPOBenchProblem + model: rf + task_id: 31 + metric: + - function_value + - cost + budget_type: null + seed: ${seed} +task: + n_trials: 100 + time_budget: null + n_workers: 1 + n_objectives: 2 + objectives: + - function_value + - cost + is_multifidelity: false + fidelity_type: null + min_budget: null + max_budget: null + has_constraints: false + domain: ML + objective_function_approximation: tabular + has_virtual_time: false + deterministic: false + dimensions: 4 + search_space_n_categoricals: 0 + search_space_n_ordinals: 4 + search_space_n_integers: 0 + search_space_n_floats: 0 + search_space_has_conditionals: false + search_space_has_forbiddens: false + search_space_has_priors: false diff --git a/carps/configs/problem/subselection/multifidelity/dev/subset_hpobench_MO_tab_ml_svm_146212.yaml b/carps/configs/problem/subselection/multifidelity/dev/subset_hpobench_MO_tab_ml_svm_146212.yaml new file mode 100644 index 00000000..16ed117e --- /dev/null +++ b/carps/configs/problem/subselection/multifidelity/dev/subset_hpobench_MO_tab_ml_svm_146212.yaml @@ -0,0 +1,37 @@ +# @package _global_ +benchmark_id: HPOBench +problem_id: multifidelity/10/dev/hpobench/MO/tab/ml/svm/146212 +problem: + _target_: carps.benchmarks.hpo_bench.HPOBenchProblem + model: svm + task_id: 146212 + metric: + - function_value + - cost + budget_type: null + seed: ${seed} +task: + n_trials: 77 + time_budget: null + n_workers: 1 + n_objectives: 2 + objectives: + - function_value + - cost + is_multifidelity: false + fidelity_type: null + min_budget: null + max_budget: null + has_constraints: false + domain: ML + objective_function_approximation: tabular + has_virtual_time: false + deterministic: false + dimensions: 2 + search_space_n_categoricals: 0 + search_space_n_ordinals: 2 + search_space_n_integers: 0 + search_space_n_floats: 0 + search_space_has_conditionals: false + search_space_has_forbiddens: false + search_space_has_priors: false diff --git a/carps/configs/problem/subselection/multifidelity/dev/subset_hpobench_MO_tab_ml_svm_146818.yaml b/carps/configs/problem/subselection/multifidelity/dev/subset_hpobench_MO_tab_ml_svm_146818.yaml new file mode 100644 index 00000000..fbb4e146 --- /dev/null +++ b/carps/configs/problem/subselection/multifidelity/dev/subset_hpobench_MO_tab_ml_svm_146818.yaml @@ -0,0 +1,37 @@ +# @package _global_ +benchmark_id: HPOBench +problem_id: multifidelity/10/dev/hpobench/MO/tab/ml/svm/146818 +problem: + _target_: carps.benchmarks.hpo_bench.HPOBenchProblem + model: svm + task_id: 146818 + metric: + - function_value + - cost + budget_type: null + seed: ${seed} +task: + n_trials: 77 + time_budget: null + n_workers: 1 + n_objectives: 2 + objectives: + - function_value + - cost + is_multifidelity: false + fidelity_type: null + min_budget: null + max_budget: null + has_constraints: false + domain: ML + objective_function_approximation: tabular + has_virtual_time: false + deterministic: false + dimensions: 2 + search_space_n_categoricals: 0 + search_space_n_ordinals: 2 + search_space_n_integers: 0 + search_space_n_floats: 0 + search_space_has_conditionals: false + search_space_has_forbiddens: false + search_space_has_priors: false diff --git a/carps/configs/problem/subselection/multifidelity/dev/subset_hpobench_MO_tab_ml_svm_31.yaml b/carps/configs/problem/subselection/multifidelity/dev/subset_hpobench_MO_tab_ml_svm_31.yaml new file mode 100644 index 00000000..fa366005 --- /dev/null +++ b/carps/configs/problem/subselection/multifidelity/dev/subset_hpobench_MO_tab_ml_svm_31.yaml @@ -0,0 +1,37 @@ +# @package _global_ +benchmark_id: HPOBench +problem_id: multifidelity/10/dev/hpobench/MO/tab/ml/svm/31 +problem: + _target_: carps.benchmarks.hpo_bench.HPOBenchProblem + model: svm + task_id: 31 + metric: + - function_value + - cost + budget_type: null + seed: ${seed} +task: + n_trials: 77 + time_budget: null + n_workers: 1 + n_objectives: 2 + objectives: + - function_value + - cost + is_multifidelity: false + fidelity_type: null + min_budget: null + max_budget: null + has_constraints: false + domain: ML + objective_function_approximation: tabular + has_virtual_time: false + deterministic: false + dimensions: 2 + search_space_n_categoricals: 0 + search_space_n_ordinals: 2 + search_space_n_integers: 0 + search_space_n_floats: 0 + search_space_has_conditionals: false + search_space_has_forbiddens: false + search_space_has_priors: false diff --git a/carps/configs/problem/subselection/multifidelity/dev/subset_hpobench_MO_tab_ml_xgboost_146822.yaml b/carps/configs/problem/subselection/multifidelity/dev/subset_hpobench_MO_tab_ml_xgboost_146822.yaml new file mode 100644 index 00000000..618702d7 --- /dev/null +++ b/carps/configs/problem/subselection/multifidelity/dev/subset_hpobench_MO_tab_ml_xgboost_146822.yaml @@ -0,0 +1,37 @@ +# @package _global_ +benchmark_id: HPOBench +problem_id: multifidelity/10/dev/hpobench/MO/tab/ml/xgboost/146822 +problem: + _target_: carps.benchmarks.hpo_bench.HPOBenchProblem + model: xgboost + task_id: 146822 + metric: + - function_value + - cost + budget_type: null + seed: ${seed} +task: + n_trials: 100 + time_budget: null + n_workers: 1 + n_objectives: 2 + objectives: + - function_value + - cost + is_multifidelity: false + fidelity_type: null + min_budget: null + max_budget: null + has_constraints: false + domain: ML + objective_function_approximation: tabular + has_virtual_time: false + deterministic: false + dimensions: 4 + search_space_n_categoricals: 0 + search_space_n_ordinals: 4 + search_space_n_integers: 0 + search_space_n_floats: 0 + search_space_has_conditionals: false + search_space_has_forbiddens: false + search_space_has_priors: false diff --git a/carps/configs/problem/subselection/multifidelity/dev/subset_hpobench_MO_tab_ml_xgboost_14965.yaml b/carps/configs/problem/subselection/multifidelity/dev/subset_hpobench_MO_tab_ml_xgboost_14965.yaml new file mode 100644 index 00000000..01e4374b --- /dev/null +++ b/carps/configs/problem/subselection/multifidelity/dev/subset_hpobench_MO_tab_ml_xgboost_14965.yaml @@ -0,0 +1,37 @@ +# @package _global_ +benchmark_id: HPOBench +problem_id: multifidelity/10/dev/hpobench/MO/tab/ml/xgboost/14965 +problem: + _target_: carps.benchmarks.hpo_bench.HPOBenchProblem + model: xgboost + task_id: 14965 + metric: + - function_value + - cost + budget_type: null + seed: ${seed} +task: + n_trials: 100 + time_budget: null + n_workers: 1 + n_objectives: 2 + objectives: + - function_value + - cost + is_multifidelity: false + fidelity_type: null + min_budget: null + max_budget: null + has_constraints: false + domain: ML + objective_function_approximation: tabular + has_virtual_time: false + deterministic: false + dimensions: 4 + search_space_n_categoricals: 0 + search_space_n_ordinals: 4 + search_space_n_integers: 0 + search_space_n_floats: 0 + search_space_has_conditionals: false + search_space_has_forbiddens: false + search_space_has_priors: false diff --git a/carps/configs/problem/subselection/multifidelity/dev/subset_hpobench_MO_tab_ml_xgboost_167120.yaml b/carps/configs/problem/subselection/multifidelity/dev/subset_hpobench_MO_tab_ml_xgboost_167120.yaml new file mode 100644 index 00000000..10d80bf2 --- /dev/null +++ b/carps/configs/problem/subselection/multifidelity/dev/subset_hpobench_MO_tab_ml_xgboost_167120.yaml @@ -0,0 +1,37 @@ +# @package _global_ +benchmark_id: HPOBench +problem_id: multifidelity/10/dev/hpobench/MO/tab/ml/xgboost/167120 +problem: + _target_: carps.benchmarks.hpo_bench.HPOBenchProblem + model: xgboost + task_id: 167120 + metric: + - function_value + - cost + budget_type: null + seed: ${seed} +task: + n_trials: 100 + time_budget: null + n_workers: 1 + n_objectives: 2 + objectives: + - function_value + - cost + is_multifidelity: false + fidelity_type: null + min_budget: null + max_budget: null + has_constraints: false + domain: ML + objective_function_approximation: tabular + has_virtual_time: false + deterministic: false + dimensions: 4 + search_space_n_categoricals: 0 + search_space_n_ordinals: 4 + search_space_n_integers: 0 + search_space_n_floats: 0 + search_space_has_conditionals: false + search_space_has_forbiddens: false + search_space_has_priors: false diff --git a/carps/configs/problem/subselection/multifidelity/dev/subset_yahpo_mo_rbv2_rpart_40499_None.yaml b/carps/configs/problem/subselection/multifidelity/dev/subset_yahpo_mo_rbv2_rpart_40499_None.yaml new file mode 100644 index 00000000..c4af5dbc --- /dev/null +++ b/carps/configs/problem/subselection/multifidelity/dev/subset_yahpo_mo_rbv2_rpart_40499_None.yaml @@ -0,0 +1,36 @@ +# @package _global_ +benchmark_id: YAHPO +problem_id: multifidelity/10/dev/yahpo/mo/rbv2_rpart/40499/None +problem: + _target_: carps.benchmarks.yahpo.YahpoProblem + bench: rbv2_rpart + instance: '40499' + budget_type: null + metric: + - acc + - memory +task: + n_trials: 110 + time_budget: null + n_workers: 1 + n_objectives: 2 + objectives: + - acc + - memory + is_multifidelity: false + fidelity_type: null + min_budget: null + max_budget: null + has_constraints: false + domain: ML + objective_function_approximation: surrogate + has_virtual_time: true + deterministic: true + dimensions: 5 + search_space_n_categoricals: 1 + search_space_n_ordinals: 0 + search_space_n_integers: 3 + search_space_n_floats: 1 + search_space_has_conditionals: false + search_space_has_forbiddens: false + search_space_has_priors: false diff --git a/carps/configs/problem/subselection/multifidelity/test/subset_Pymoo_MO_zdt3.yaml b/carps/configs/problem/subselection/multifidelity/test/subset_Pymoo_MO_zdt3.yaml new file mode 100644 index 00000000..41d4766c --- /dev/null +++ b/carps/configs/problem/subselection/multifidelity/test/subset_Pymoo_MO_zdt3.yaml @@ -0,0 +1,35 @@ +# @package _global_ +benchmark_id: Pymoo +problem_id: multifidelity/10/test/Pymoo/MO/zdt3 +problem: + _target_: carps.benchmarks.pymoo.PymooProblem + problem_name: zdt3 + seed: ${seed} + metric: + - objective_0 + - objective_1 +task: + n_trials: 240 + time_budget: null + n_workers: 1 + n_objectives: 2 + objectives: + - objective_0 + - objective_1 + is_multifidelity: false + fidelity_type: null + min_budget: null + max_budget: null + has_constraints: false + domain: synthetic + objective_function_approximation: real + has_virtual_time: false + deterministic: true + dimensions: 30 + search_space_n_categoricals: 0 + search_space_n_ordinals: 0 + search_space_n_integers: 0 + search_space_n_floats: 30 + search_space_has_conditionals: false + search_space_has_forbiddens: false + search_space_has_priors: false diff --git a/carps/configs/problem/subselection/multifidelity/test/subset_hpobench_MO_tab_ml_lr_146606.yaml b/carps/configs/problem/subselection/multifidelity/test/subset_hpobench_MO_tab_ml_lr_146606.yaml new file mode 100644 index 00000000..cf0e2827 --- /dev/null +++ b/carps/configs/problem/subselection/multifidelity/test/subset_hpobench_MO_tab_ml_lr_146606.yaml @@ -0,0 +1,37 @@ +# @package _global_ +benchmark_id: HPOBench +problem_id: multifidelity/10/test/hpobench/MO/tab/ml/lr/146606 +problem: + _target_: carps.benchmarks.hpo_bench.HPOBenchProblem + model: lr + task_id: 146606 + metric: + - function_value + - cost + budget_type: null + seed: ${seed} +task: + n_trials: 77 + time_budget: null + n_workers: 1 + n_objectives: 2 + objectives: + - function_value + - cost + is_multifidelity: false + fidelity_type: null + min_budget: null + max_budget: null + has_constraints: false + domain: ML + objective_function_approximation: tabular + has_virtual_time: false + deterministic: false + dimensions: 2 + search_space_n_categoricals: 0 + search_space_n_ordinals: 2 + search_space_n_integers: 0 + search_space_n_floats: 0 + search_space_has_conditionals: false + search_space_has_forbiddens: false + search_space_has_priors: false diff --git a/carps/configs/problem/subselection/multifidelity/test/subset_hpobench_MO_tab_ml_lr_167119.yaml b/carps/configs/problem/subselection/multifidelity/test/subset_hpobench_MO_tab_ml_lr_167119.yaml new file mode 100644 index 00000000..5a1c88ef --- /dev/null +++ b/carps/configs/problem/subselection/multifidelity/test/subset_hpobench_MO_tab_ml_lr_167119.yaml @@ -0,0 +1,37 @@ +# @package _global_ +benchmark_id: HPOBench +problem_id: multifidelity/10/test/hpobench/MO/tab/ml/lr/167119 +problem: + _target_: carps.benchmarks.hpo_bench.HPOBenchProblem + model: lr + task_id: 167119 + metric: + - function_value + - cost + budget_type: null + seed: ${seed} +task: + n_trials: 77 + time_budget: null + n_workers: 1 + n_objectives: 2 + objectives: + - function_value + - cost + is_multifidelity: false + fidelity_type: null + min_budget: null + max_budget: null + has_constraints: false + domain: ML + objective_function_approximation: tabular + has_virtual_time: false + deterministic: false + dimensions: 2 + search_space_n_categoricals: 0 + search_space_n_ordinals: 2 + search_space_n_integers: 0 + search_space_n_floats: 0 + search_space_has_conditionals: false + search_space_has_forbiddens: false + search_space_has_priors: false diff --git a/carps/configs/problem/subselection/multifidelity/test/subset_hpobench_MO_tab_ml_lr_168912.yaml b/carps/configs/problem/subselection/multifidelity/test/subset_hpobench_MO_tab_ml_lr_168912.yaml new file mode 100644 index 00000000..88d1dfd4 --- /dev/null +++ b/carps/configs/problem/subselection/multifidelity/test/subset_hpobench_MO_tab_ml_lr_168912.yaml @@ -0,0 +1,37 @@ +# @package _global_ +benchmark_id: HPOBench +problem_id: multifidelity/10/test/hpobench/MO/tab/ml/lr/168912 +problem: + _target_: carps.benchmarks.hpo_bench.HPOBenchProblem + model: lr + task_id: 168912 + metric: + - function_value + - cost + budget_type: null + seed: ${seed} +task: + n_trials: 77 + time_budget: null + n_workers: 1 + n_objectives: 2 + objectives: + - function_value + - cost + is_multifidelity: false + fidelity_type: null + min_budget: null + max_budget: null + has_constraints: false + domain: ML + objective_function_approximation: tabular + has_virtual_time: false + deterministic: false + dimensions: 2 + search_space_n_categoricals: 0 + search_space_n_ordinals: 2 + search_space_n_integers: 0 + search_space_n_floats: 0 + search_space_has_conditionals: false + search_space_has_forbiddens: false + search_space_has_priors: false diff --git a/carps/configs/problem/subselection/multifidelity/test/subset_hpobench_MO_tab_ml_nn_146821.yaml b/carps/configs/problem/subselection/multifidelity/test/subset_hpobench_MO_tab_ml_nn_146821.yaml new file mode 100644 index 00000000..298fd5db --- /dev/null +++ b/carps/configs/problem/subselection/multifidelity/test/subset_hpobench_MO_tab_ml_nn_146821.yaml @@ -0,0 +1,37 @@ +# @package _global_ +benchmark_id: HPOBench +problem_id: multifidelity/10/test/hpobench/MO/tab/ml/nn/146821 +problem: + _target_: carps.benchmarks.hpo_bench.HPOBenchProblem + model: nn + task_id: 146821 + metric: + - function_value + - cost + budget_type: null + seed: ${seed} +task: + n_trials: 110 + time_budget: null + n_workers: 1 + n_objectives: 2 + objectives: + - function_value + - cost + is_multifidelity: false + fidelity_type: null + min_budget: null + max_budget: null + has_constraints: false + domain: ML + objective_function_approximation: tabular + has_virtual_time: false + deterministic: false + dimensions: 5 + search_space_n_categoricals: 0 + search_space_n_ordinals: 5 + search_space_n_integers: 0 + search_space_n_floats: 0 + search_space_has_conditionals: false + search_space_has_forbiddens: false + search_space_has_priors: false diff --git a/carps/configs/problem/subselection/multifidelity/test/subset_hpobench_MO_tab_ml_rf_146212.yaml b/carps/configs/problem/subselection/multifidelity/test/subset_hpobench_MO_tab_ml_rf_146212.yaml new file mode 100644 index 00000000..f43393e5 --- /dev/null +++ b/carps/configs/problem/subselection/multifidelity/test/subset_hpobench_MO_tab_ml_rf_146212.yaml @@ -0,0 +1,37 @@ +# @package _global_ +benchmark_id: HPOBench +problem_id: multifidelity/10/test/hpobench/MO/tab/ml/rf/146212 +problem: + _target_: carps.benchmarks.hpo_bench.HPOBenchProblem + model: rf + task_id: 146212 + metric: + - function_value + - cost + budget_type: null + seed: ${seed} +task: + n_trials: 100 + time_budget: null + n_workers: 1 + n_objectives: 2 + objectives: + - function_value + - cost + is_multifidelity: false + fidelity_type: null + min_budget: null + max_budget: null + has_constraints: false + domain: ML + objective_function_approximation: tabular + has_virtual_time: false + deterministic: false + dimensions: 4 + search_space_n_categoricals: 0 + search_space_n_ordinals: 4 + search_space_n_integers: 0 + search_space_n_floats: 0 + search_space_has_conditionals: false + search_space_has_forbiddens: false + search_space_has_priors: false diff --git a/carps/configs/problem/subselection/multifidelity/test/subset_hpobench_MO_tab_ml_svm_10101.yaml b/carps/configs/problem/subselection/multifidelity/test/subset_hpobench_MO_tab_ml_svm_10101.yaml new file mode 100644 index 00000000..273da0c7 --- /dev/null +++ b/carps/configs/problem/subselection/multifidelity/test/subset_hpobench_MO_tab_ml_svm_10101.yaml @@ -0,0 +1,37 @@ +# @package _global_ +benchmark_id: HPOBench +problem_id: multifidelity/10/test/hpobench/MO/tab/ml/svm/10101 +problem: + _target_: carps.benchmarks.hpo_bench.HPOBenchProblem + model: svm + task_id: 10101 + metric: + - function_value + - cost + budget_type: null + seed: ${seed} +task: + n_trials: 77 + time_budget: null + n_workers: 1 + n_objectives: 2 + objectives: + - function_value + - cost + is_multifidelity: false + fidelity_type: null + min_budget: null + max_budget: null + has_constraints: false + domain: ML + objective_function_approximation: tabular + has_virtual_time: false + deterministic: false + dimensions: 2 + search_space_n_categoricals: 0 + search_space_n_ordinals: 2 + search_space_n_integers: 0 + search_space_n_floats: 0 + search_space_has_conditionals: false + search_space_has_forbiddens: false + search_space_has_priors: false diff --git a/carps/configs/problem/subselection/multifidelity/test/subset_hpobench_MO_tab_ml_svm_146821.yaml b/carps/configs/problem/subselection/multifidelity/test/subset_hpobench_MO_tab_ml_svm_146821.yaml new file mode 100644 index 00000000..80a176da --- /dev/null +++ b/carps/configs/problem/subselection/multifidelity/test/subset_hpobench_MO_tab_ml_svm_146821.yaml @@ -0,0 +1,37 @@ +# @package _global_ +benchmark_id: HPOBench +problem_id: multifidelity/10/test/hpobench/MO/tab/ml/svm/146821 +problem: + _target_: carps.benchmarks.hpo_bench.HPOBenchProblem + model: svm + task_id: 146821 + metric: + - function_value + - cost + budget_type: null + seed: ${seed} +task: + n_trials: 77 + time_budget: null + n_workers: 1 + n_objectives: 2 + objectives: + - function_value + - cost + is_multifidelity: false + fidelity_type: null + min_budget: null + max_budget: null + has_constraints: false + domain: ML + objective_function_approximation: tabular + has_virtual_time: false + deterministic: false + dimensions: 2 + search_space_n_categoricals: 0 + search_space_n_ordinals: 2 + search_space_n_integers: 0 + search_space_n_floats: 0 + search_space_has_conditionals: false + search_space_has_forbiddens: false + search_space_has_priors: false diff --git a/carps/configs/problem/subselection/multifidelity/test/subset_hpobench_MO_tab_ml_svm_7592.yaml b/carps/configs/problem/subselection/multifidelity/test/subset_hpobench_MO_tab_ml_svm_7592.yaml new file mode 100644 index 00000000..74f250ef --- /dev/null +++ b/carps/configs/problem/subselection/multifidelity/test/subset_hpobench_MO_tab_ml_svm_7592.yaml @@ -0,0 +1,37 @@ +# @package _global_ +benchmark_id: HPOBench +problem_id: multifidelity/10/test/hpobench/MO/tab/ml/svm/7592 +problem: + _target_: carps.benchmarks.hpo_bench.HPOBenchProblem + model: svm + task_id: 7592 + metric: + - function_value + - cost + budget_type: null + seed: ${seed} +task: + n_trials: 77 + time_budget: null + n_workers: 1 + n_objectives: 2 + objectives: + - function_value + - cost + is_multifidelity: false + fidelity_type: null + min_budget: null + max_budget: null + has_constraints: false + domain: ML + objective_function_approximation: tabular + has_virtual_time: false + deterministic: false + dimensions: 2 + search_space_n_categoricals: 0 + search_space_n_ordinals: 2 + search_space_n_integers: 0 + search_space_n_floats: 0 + search_space_has_conditionals: false + search_space_has_forbiddens: false + search_space_has_priors: false diff --git a/carps/configs/problem/subselection/multifidelity/test/subset_hpobench_MO_tab_ml_xgboost_168911.yaml b/carps/configs/problem/subselection/multifidelity/test/subset_hpobench_MO_tab_ml_xgboost_168911.yaml new file mode 100644 index 00000000..75a35f36 --- /dev/null +++ b/carps/configs/problem/subselection/multifidelity/test/subset_hpobench_MO_tab_ml_xgboost_168911.yaml @@ -0,0 +1,37 @@ +# @package _global_ +benchmark_id: HPOBench +problem_id: multifidelity/10/test/hpobench/MO/tab/ml/xgboost/168911 +problem: + _target_: carps.benchmarks.hpo_bench.HPOBenchProblem + model: xgboost + task_id: 168911 + metric: + - function_value + - cost + budget_type: null + seed: ${seed} +task: + n_trials: 100 + time_budget: null + n_workers: 1 + n_objectives: 2 + objectives: + - function_value + - cost + is_multifidelity: false + fidelity_type: null + min_budget: null + max_budget: null + has_constraints: false + domain: ML + objective_function_approximation: tabular + has_virtual_time: false + deterministic: false + dimensions: 4 + search_space_n_categoricals: 0 + search_space_n_ordinals: 4 + search_space_n_integers: 0 + search_space_n_floats: 0 + search_space_has_conditionals: false + search_space_has_forbiddens: false + search_space_has_priors: false diff --git a/carps/configs/problem/subselection/multiobjective/dev/subset_Pymoo_MO_omnitest.yaml b/carps/configs/problem/subselection/multiobjective/dev/subset_Pymoo_MO_omnitest.yaml new file mode 100644 index 00000000..47baf17b --- /dev/null +++ b/carps/configs/problem/subselection/multiobjective/dev/subset_Pymoo_MO_omnitest.yaml @@ -0,0 +1,35 @@ +# @package _global_ +benchmark_id: Pymoo +problem_id: multiobjective/10/dev/Pymoo/MO/omnitest +problem: + _target_: carps.benchmarks.pymoo.PymooProblem + problem_name: omnitest + seed: ${seed} + metric: + - objective_0 + - objective_1 +task: + n_trials: 77 + time_budget: null + n_workers: 1 + n_objectives: 2 + objectives: + - objective_0 + - objective_1 + is_multifidelity: false + fidelity_type: null + min_budget: null + max_budget: null + has_constraints: false + domain: synthetic + objective_function_approximation: real + has_virtual_time: false + deterministic: true + dimensions: 2 + search_space_n_categoricals: 0 + search_space_n_ordinals: 0 + search_space_n_integers: 0 + search_space_n_floats: 2 + search_space_has_conditionals: false + search_space_has_forbiddens: false + search_space_has_priors: false diff --git a/carps/configs/problem/subselection/multiobjective/dev/subset_hpobench_MO_tab_ml_rf_167119.yaml b/carps/configs/problem/subselection/multiobjective/dev/subset_hpobench_MO_tab_ml_rf_167119.yaml new file mode 100644 index 00000000..37e891f5 --- /dev/null +++ b/carps/configs/problem/subselection/multiobjective/dev/subset_hpobench_MO_tab_ml_rf_167119.yaml @@ -0,0 +1,37 @@ +# @package _global_ +benchmark_id: HPOBench +problem_id: multiobjective/10/dev/hpobench/MO/tab/ml/rf/167119 +problem: + _target_: carps.benchmarks.hpo_bench.HPOBenchProblem + model: rf + task_id: 167119 + metric: + - function_value + - cost + budget_type: null + seed: ${seed} +task: + n_trials: 100 + time_budget: null + n_workers: 1 + n_objectives: 2 + objectives: + - function_value + - cost + is_multifidelity: false + fidelity_type: null + min_budget: null + max_budget: null + has_constraints: false + domain: ML + objective_function_approximation: tabular + has_virtual_time: false + deterministic: false + dimensions: 4 + search_space_n_categoricals: 0 + search_space_n_ordinals: 4 + search_space_n_integers: 0 + search_space_n_floats: 0 + search_space_has_conditionals: false + search_space_has_forbiddens: false + search_space_has_priors: false diff --git a/carps/configs/problem/subselection/multiobjective/dev/subset_hpobench_MO_tab_ml_rf_31.yaml b/carps/configs/problem/subselection/multiobjective/dev/subset_hpobench_MO_tab_ml_rf_31.yaml new file mode 100644 index 00000000..1fff6c76 --- /dev/null +++ b/carps/configs/problem/subselection/multiobjective/dev/subset_hpobench_MO_tab_ml_rf_31.yaml @@ -0,0 +1,37 @@ +# @package _global_ +benchmark_id: HPOBench +problem_id: multiobjective/10/dev/hpobench/MO/tab/ml/rf/31 +problem: + _target_: carps.benchmarks.hpo_bench.HPOBenchProblem + model: rf + task_id: 31 + metric: + - function_value + - cost + budget_type: null + seed: ${seed} +task: + n_trials: 100 + time_budget: null + n_workers: 1 + n_objectives: 2 + objectives: + - function_value + - cost + is_multifidelity: false + fidelity_type: null + min_budget: null + max_budget: null + has_constraints: false + domain: ML + objective_function_approximation: tabular + has_virtual_time: false + deterministic: false + dimensions: 4 + search_space_n_categoricals: 0 + search_space_n_ordinals: 4 + search_space_n_integers: 0 + search_space_n_floats: 0 + search_space_has_conditionals: false + search_space_has_forbiddens: false + search_space_has_priors: false diff --git a/carps/configs/problem/subselection/multiobjective/dev/subset_hpobench_MO_tab_ml_svm_146212.yaml b/carps/configs/problem/subselection/multiobjective/dev/subset_hpobench_MO_tab_ml_svm_146212.yaml new file mode 100644 index 00000000..da98c184 --- /dev/null +++ b/carps/configs/problem/subselection/multiobjective/dev/subset_hpobench_MO_tab_ml_svm_146212.yaml @@ -0,0 +1,37 @@ +# @package _global_ +benchmark_id: HPOBench +problem_id: multiobjective/10/dev/hpobench/MO/tab/ml/svm/146212 +problem: + _target_: carps.benchmarks.hpo_bench.HPOBenchProblem + model: svm + task_id: 146212 + metric: + - function_value + - cost + budget_type: null + seed: ${seed} +task: + n_trials: 77 + time_budget: null + n_workers: 1 + n_objectives: 2 + objectives: + - function_value + - cost + is_multifidelity: false + fidelity_type: null + min_budget: null + max_budget: null + has_constraints: false + domain: ML + objective_function_approximation: tabular + has_virtual_time: false + deterministic: false + dimensions: 2 + search_space_n_categoricals: 0 + search_space_n_ordinals: 2 + search_space_n_integers: 0 + search_space_n_floats: 0 + search_space_has_conditionals: false + search_space_has_forbiddens: false + search_space_has_priors: false diff --git a/carps/configs/problem/subselection/multiobjective/dev/subset_hpobench_MO_tab_ml_svm_146818.yaml b/carps/configs/problem/subselection/multiobjective/dev/subset_hpobench_MO_tab_ml_svm_146818.yaml new file mode 100644 index 00000000..99995a3d --- /dev/null +++ b/carps/configs/problem/subselection/multiobjective/dev/subset_hpobench_MO_tab_ml_svm_146818.yaml @@ -0,0 +1,37 @@ +# @package _global_ +benchmark_id: HPOBench +problem_id: multiobjective/10/dev/hpobench/MO/tab/ml/svm/146818 +problem: + _target_: carps.benchmarks.hpo_bench.HPOBenchProblem + model: svm + task_id: 146818 + metric: + - function_value + - cost + budget_type: null + seed: ${seed} +task: + n_trials: 77 + time_budget: null + n_workers: 1 + n_objectives: 2 + objectives: + - function_value + - cost + is_multifidelity: false + fidelity_type: null + min_budget: null + max_budget: null + has_constraints: false + domain: ML + objective_function_approximation: tabular + has_virtual_time: false + deterministic: false + dimensions: 2 + search_space_n_categoricals: 0 + search_space_n_ordinals: 2 + search_space_n_integers: 0 + search_space_n_floats: 0 + search_space_has_conditionals: false + search_space_has_forbiddens: false + search_space_has_priors: false diff --git a/carps/configs/problem/subselection/multiobjective/dev/subset_hpobench_MO_tab_ml_svm_31.yaml b/carps/configs/problem/subselection/multiobjective/dev/subset_hpobench_MO_tab_ml_svm_31.yaml new file mode 100644 index 00000000..191a17e7 --- /dev/null +++ b/carps/configs/problem/subselection/multiobjective/dev/subset_hpobench_MO_tab_ml_svm_31.yaml @@ -0,0 +1,37 @@ +# @package _global_ +benchmark_id: HPOBench +problem_id: multiobjective/10/dev/hpobench/MO/tab/ml/svm/31 +problem: + _target_: carps.benchmarks.hpo_bench.HPOBenchProblem + model: svm + task_id: 31 + metric: + - function_value + - cost + budget_type: null + seed: ${seed} +task: + n_trials: 77 + time_budget: null + n_workers: 1 + n_objectives: 2 + objectives: + - function_value + - cost + is_multifidelity: false + fidelity_type: null + min_budget: null + max_budget: null + has_constraints: false + domain: ML + objective_function_approximation: tabular + has_virtual_time: false + deterministic: false + dimensions: 2 + search_space_n_categoricals: 0 + search_space_n_ordinals: 2 + search_space_n_integers: 0 + search_space_n_floats: 0 + search_space_has_conditionals: false + search_space_has_forbiddens: false + search_space_has_priors: false diff --git a/carps/configs/problem/subselection/multiobjective/dev/subset_hpobench_MO_tab_ml_xgboost_146822.yaml b/carps/configs/problem/subselection/multiobjective/dev/subset_hpobench_MO_tab_ml_xgboost_146822.yaml new file mode 100644 index 00000000..cb3d5a2e --- /dev/null +++ b/carps/configs/problem/subselection/multiobjective/dev/subset_hpobench_MO_tab_ml_xgboost_146822.yaml @@ -0,0 +1,37 @@ +# @package _global_ +benchmark_id: HPOBench +problem_id: multiobjective/10/dev/hpobench/MO/tab/ml/xgboost/146822 +problem: + _target_: carps.benchmarks.hpo_bench.HPOBenchProblem + model: xgboost + task_id: 146822 + metric: + - function_value + - cost + budget_type: null + seed: ${seed} +task: + n_trials: 100 + time_budget: null + n_workers: 1 + n_objectives: 2 + objectives: + - function_value + - cost + is_multifidelity: false + fidelity_type: null + min_budget: null + max_budget: null + has_constraints: false + domain: ML + objective_function_approximation: tabular + has_virtual_time: false + deterministic: false + dimensions: 4 + search_space_n_categoricals: 0 + search_space_n_ordinals: 4 + search_space_n_integers: 0 + search_space_n_floats: 0 + search_space_has_conditionals: false + search_space_has_forbiddens: false + search_space_has_priors: false diff --git a/carps/configs/problem/subselection/multiobjective/dev/subset_hpobench_MO_tab_ml_xgboost_14965.yaml b/carps/configs/problem/subselection/multiobjective/dev/subset_hpobench_MO_tab_ml_xgboost_14965.yaml new file mode 100644 index 00000000..cf8d72ab --- /dev/null +++ b/carps/configs/problem/subselection/multiobjective/dev/subset_hpobench_MO_tab_ml_xgboost_14965.yaml @@ -0,0 +1,37 @@ +# @package _global_ +benchmark_id: HPOBench +problem_id: multiobjective/10/dev/hpobench/MO/tab/ml/xgboost/14965 +problem: + _target_: carps.benchmarks.hpo_bench.HPOBenchProblem + model: xgboost + task_id: 14965 + metric: + - function_value + - cost + budget_type: null + seed: ${seed} +task: + n_trials: 100 + time_budget: null + n_workers: 1 + n_objectives: 2 + objectives: + - function_value + - cost + is_multifidelity: false + fidelity_type: null + min_budget: null + max_budget: null + has_constraints: false + domain: ML + objective_function_approximation: tabular + has_virtual_time: false + deterministic: false + dimensions: 4 + search_space_n_categoricals: 0 + search_space_n_ordinals: 4 + search_space_n_integers: 0 + search_space_n_floats: 0 + search_space_has_conditionals: false + search_space_has_forbiddens: false + search_space_has_priors: false diff --git a/carps/configs/problem/subselection/multiobjective/dev/subset_hpobench_MO_tab_ml_xgboost_167120.yaml b/carps/configs/problem/subselection/multiobjective/dev/subset_hpobench_MO_tab_ml_xgboost_167120.yaml new file mode 100644 index 00000000..af01acd3 --- /dev/null +++ b/carps/configs/problem/subselection/multiobjective/dev/subset_hpobench_MO_tab_ml_xgboost_167120.yaml @@ -0,0 +1,37 @@ +# @package _global_ +benchmark_id: HPOBench +problem_id: multiobjective/10/dev/hpobench/MO/tab/ml/xgboost/167120 +problem: + _target_: carps.benchmarks.hpo_bench.HPOBenchProblem + model: xgboost + task_id: 167120 + metric: + - function_value + - cost + budget_type: null + seed: ${seed} +task: + n_trials: 100 + time_budget: null + n_workers: 1 + n_objectives: 2 + objectives: + - function_value + - cost + is_multifidelity: false + fidelity_type: null + min_budget: null + max_budget: null + has_constraints: false + domain: ML + objective_function_approximation: tabular + has_virtual_time: false + deterministic: false + dimensions: 4 + search_space_n_categoricals: 0 + search_space_n_ordinals: 4 + search_space_n_integers: 0 + search_space_n_floats: 0 + search_space_has_conditionals: false + search_space_has_forbiddens: false + search_space_has_priors: false diff --git a/carps/configs/problem/subselection/multiobjective/dev/subset_yahpo_mo_rbv2_rpart_40499_None.yaml b/carps/configs/problem/subselection/multiobjective/dev/subset_yahpo_mo_rbv2_rpart_40499_None.yaml new file mode 100644 index 00000000..08f3d015 --- /dev/null +++ b/carps/configs/problem/subselection/multiobjective/dev/subset_yahpo_mo_rbv2_rpart_40499_None.yaml @@ -0,0 +1,36 @@ +# @package _global_ +benchmark_id: YAHPO +problem_id: multiobjective/10/dev/yahpo/mo/rbv2_rpart/40499/None +problem: + _target_: carps.benchmarks.yahpo.YahpoProblem + bench: rbv2_rpart + instance: '40499' + budget_type: null + metric: + - acc + - memory +task: + n_trials: 110 + time_budget: null + n_workers: 1 + n_objectives: 2 + objectives: + - acc + - memory + is_multifidelity: false + fidelity_type: null + min_budget: null + max_budget: null + has_constraints: false + domain: ML + objective_function_approximation: surrogate + has_virtual_time: true + deterministic: true + dimensions: 5 + search_space_n_categoricals: 1 + search_space_n_ordinals: 0 + search_space_n_integers: 3 + search_space_n_floats: 1 + search_space_has_conditionals: false + search_space_has_forbiddens: false + search_space_has_priors: false diff --git a/carps/configs/problem/subselection/multiobjective/test/subset_Pymoo_MO_zdt3.yaml b/carps/configs/problem/subselection/multiobjective/test/subset_Pymoo_MO_zdt3.yaml new file mode 100644 index 00000000..636bbf3c --- /dev/null +++ b/carps/configs/problem/subselection/multiobjective/test/subset_Pymoo_MO_zdt3.yaml @@ -0,0 +1,35 @@ +# @package _global_ +benchmark_id: Pymoo +problem_id: multiobjective/10/test/Pymoo/MO/zdt3 +problem: + _target_: carps.benchmarks.pymoo.PymooProblem + problem_name: zdt3 + seed: ${seed} + metric: + - objective_0 + - objective_1 +task: + n_trials: 240 + time_budget: null + n_workers: 1 + n_objectives: 2 + objectives: + - objective_0 + - objective_1 + is_multifidelity: false + fidelity_type: null + min_budget: null + max_budget: null + has_constraints: false + domain: synthetic + objective_function_approximation: real + has_virtual_time: false + deterministic: true + dimensions: 30 + search_space_n_categoricals: 0 + search_space_n_ordinals: 0 + search_space_n_integers: 0 + search_space_n_floats: 30 + search_space_has_conditionals: false + search_space_has_forbiddens: false + search_space_has_priors: false diff --git a/carps/configs/problem/subselection/multiobjective/test/subset_hpobench_MO_tab_ml_lr_146606.yaml b/carps/configs/problem/subselection/multiobjective/test/subset_hpobench_MO_tab_ml_lr_146606.yaml new file mode 100644 index 00000000..3284819f --- /dev/null +++ b/carps/configs/problem/subselection/multiobjective/test/subset_hpobench_MO_tab_ml_lr_146606.yaml @@ -0,0 +1,37 @@ +# @package _global_ +benchmark_id: HPOBench +problem_id: multiobjective/10/test/hpobench/MO/tab/ml/lr/146606 +problem: + _target_: carps.benchmarks.hpo_bench.HPOBenchProblem + model: lr + task_id: 146606 + metric: + - function_value + - cost + budget_type: null + seed: ${seed} +task: + n_trials: 77 + time_budget: null + n_workers: 1 + n_objectives: 2 + objectives: + - function_value + - cost + is_multifidelity: false + fidelity_type: null + min_budget: null + max_budget: null + has_constraints: false + domain: ML + objective_function_approximation: tabular + has_virtual_time: false + deterministic: false + dimensions: 2 + search_space_n_categoricals: 0 + search_space_n_ordinals: 2 + search_space_n_integers: 0 + search_space_n_floats: 0 + search_space_has_conditionals: false + search_space_has_forbiddens: false + search_space_has_priors: false diff --git a/carps/configs/problem/subselection/multiobjective/test/subset_hpobench_MO_tab_ml_lr_167119.yaml b/carps/configs/problem/subselection/multiobjective/test/subset_hpobench_MO_tab_ml_lr_167119.yaml new file mode 100644 index 00000000..7f66347f --- /dev/null +++ b/carps/configs/problem/subselection/multiobjective/test/subset_hpobench_MO_tab_ml_lr_167119.yaml @@ -0,0 +1,37 @@ +# @package _global_ +benchmark_id: HPOBench +problem_id: multiobjective/10/test/hpobench/MO/tab/ml/lr/167119 +problem: + _target_: carps.benchmarks.hpo_bench.HPOBenchProblem + model: lr + task_id: 167119 + metric: + - function_value + - cost + budget_type: null + seed: ${seed} +task: + n_trials: 77 + time_budget: null + n_workers: 1 + n_objectives: 2 + objectives: + - function_value + - cost + is_multifidelity: false + fidelity_type: null + min_budget: null + max_budget: null + has_constraints: false + domain: ML + objective_function_approximation: tabular + has_virtual_time: false + deterministic: false + dimensions: 2 + search_space_n_categoricals: 0 + search_space_n_ordinals: 2 + search_space_n_integers: 0 + search_space_n_floats: 0 + search_space_has_conditionals: false + search_space_has_forbiddens: false + search_space_has_priors: false diff --git a/carps/configs/problem/subselection/multiobjective/test/subset_hpobench_MO_tab_ml_lr_168912.yaml b/carps/configs/problem/subselection/multiobjective/test/subset_hpobench_MO_tab_ml_lr_168912.yaml new file mode 100644 index 00000000..c2d75d0d --- /dev/null +++ b/carps/configs/problem/subselection/multiobjective/test/subset_hpobench_MO_tab_ml_lr_168912.yaml @@ -0,0 +1,37 @@ +# @package _global_ +benchmark_id: HPOBench +problem_id: multiobjective/10/test/hpobench/MO/tab/ml/lr/168912 +problem: + _target_: carps.benchmarks.hpo_bench.HPOBenchProblem + model: lr + task_id: 168912 + metric: + - function_value + - cost + budget_type: null + seed: ${seed} +task: + n_trials: 77 + time_budget: null + n_workers: 1 + n_objectives: 2 + objectives: + - function_value + - cost + is_multifidelity: false + fidelity_type: null + min_budget: null + max_budget: null + has_constraints: false + domain: ML + objective_function_approximation: tabular + has_virtual_time: false + deterministic: false + dimensions: 2 + search_space_n_categoricals: 0 + search_space_n_ordinals: 2 + search_space_n_integers: 0 + search_space_n_floats: 0 + search_space_has_conditionals: false + search_space_has_forbiddens: false + search_space_has_priors: false diff --git a/carps/configs/problem/subselection/multiobjective/test/subset_hpobench_MO_tab_ml_nn_146821.yaml b/carps/configs/problem/subselection/multiobjective/test/subset_hpobench_MO_tab_ml_nn_146821.yaml new file mode 100644 index 00000000..722d4dd1 --- /dev/null +++ b/carps/configs/problem/subselection/multiobjective/test/subset_hpobench_MO_tab_ml_nn_146821.yaml @@ -0,0 +1,37 @@ +# @package _global_ +benchmark_id: HPOBench +problem_id: multiobjective/10/test/hpobench/MO/tab/ml/nn/146821 +problem: + _target_: carps.benchmarks.hpo_bench.HPOBenchProblem + model: nn + task_id: 146821 + metric: + - function_value + - cost + budget_type: null + seed: ${seed} +task: + n_trials: 110 + time_budget: null + n_workers: 1 + n_objectives: 2 + objectives: + - function_value + - cost + is_multifidelity: false + fidelity_type: null + min_budget: null + max_budget: null + has_constraints: false + domain: ML + objective_function_approximation: tabular + has_virtual_time: false + deterministic: false + dimensions: 5 + search_space_n_categoricals: 0 + search_space_n_ordinals: 5 + search_space_n_integers: 0 + search_space_n_floats: 0 + search_space_has_conditionals: false + search_space_has_forbiddens: false + search_space_has_priors: false diff --git a/carps/configs/problem/subselection/multiobjective/test/subset_hpobench_MO_tab_ml_rf_146212.yaml b/carps/configs/problem/subselection/multiobjective/test/subset_hpobench_MO_tab_ml_rf_146212.yaml new file mode 100644 index 00000000..eefe8450 --- /dev/null +++ b/carps/configs/problem/subselection/multiobjective/test/subset_hpobench_MO_tab_ml_rf_146212.yaml @@ -0,0 +1,37 @@ +# @package _global_ +benchmark_id: HPOBench +problem_id: multiobjective/10/test/hpobench/MO/tab/ml/rf/146212 +problem: + _target_: carps.benchmarks.hpo_bench.HPOBenchProblem + model: rf + task_id: 146212 + metric: + - function_value + - cost + budget_type: null + seed: ${seed} +task: + n_trials: 100 + time_budget: null + n_workers: 1 + n_objectives: 2 + objectives: + - function_value + - cost + is_multifidelity: false + fidelity_type: null + min_budget: null + max_budget: null + has_constraints: false + domain: ML + objective_function_approximation: tabular + has_virtual_time: false + deterministic: false + dimensions: 4 + search_space_n_categoricals: 0 + search_space_n_ordinals: 4 + search_space_n_integers: 0 + search_space_n_floats: 0 + search_space_has_conditionals: false + search_space_has_forbiddens: false + search_space_has_priors: false diff --git a/carps/configs/problem/subselection/multiobjective/test/subset_hpobench_MO_tab_ml_svm_10101.yaml b/carps/configs/problem/subselection/multiobjective/test/subset_hpobench_MO_tab_ml_svm_10101.yaml new file mode 100644 index 00000000..b63fa686 --- /dev/null +++ b/carps/configs/problem/subselection/multiobjective/test/subset_hpobench_MO_tab_ml_svm_10101.yaml @@ -0,0 +1,37 @@ +# @package _global_ +benchmark_id: HPOBench +problem_id: multiobjective/10/test/hpobench/MO/tab/ml/svm/10101 +problem: + _target_: carps.benchmarks.hpo_bench.HPOBenchProblem + model: svm + task_id: 10101 + metric: + - function_value + - cost + budget_type: null + seed: ${seed} +task: + n_trials: 77 + time_budget: null + n_workers: 1 + n_objectives: 2 + objectives: + - function_value + - cost + is_multifidelity: false + fidelity_type: null + min_budget: null + max_budget: null + has_constraints: false + domain: ML + objective_function_approximation: tabular + has_virtual_time: false + deterministic: false + dimensions: 2 + search_space_n_categoricals: 0 + search_space_n_ordinals: 2 + search_space_n_integers: 0 + search_space_n_floats: 0 + search_space_has_conditionals: false + search_space_has_forbiddens: false + search_space_has_priors: false diff --git a/carps/configs/problem/subselection/multiobjective/test/subset_hpobench_MO_tab_ml_svm_146821.yaml b/carps/configs/problem/subselection/multiobjective/test/subset_hpobench_MO_tab_ml_svm_146821.yaml new file mode 100644 index 00000000..8de35e4e --- /dev/null +++ b/carps/configs/problem/subselection/multiobjective/test/subset_hpobench_MO_tab_ml_svm_146821.yaml @@ -0,0 +1,37 @@ +# @package _global_ +benchmark_id: HPOBench +problem_id: multiobjective/10/test/hpobench/MO/tab/ml/svm/146821 +problem: + _target_: carps.benchmarks.hpo_bench.HPOBenchProblem + model: svm + task_id: 146821 + metric: + - function_value + - cost + budget_type: null + seed: ${seed} +task: + n_trials: 77 + time_budget: null + n_workers: 1 + n_objectives: 2 + objectives: + - function_value + - cost + is_multifidelity: false + fidelity_type: null + min_budget: null + max_budget: null + has_constraints: false + domain: ML + objective_function_approximation: tabular + has_virtual_time: false + deterministic: false + dimensions: 2 + search_space_n_categoricals: 0 + search_space_n_ordinals: 2 + search_space_n_integers: 0 + search_space_n_floats: 0 + search_space_has_conditionals: false + search_space_has_forbiddens: false + search_space_has_priors: false diff --git a/carps/configs/problem/subselection/multiobjective/test/subset_hpobench_MO_tab_ml_svm_7592.yaml b/carps/configs/problem/subselection/multiobjective/test/subset_hpobench_MO_tab_ml_svm_7592.yaml new file mode 100644 index 00000000..0c36f362 --- /dev/null +++ b/carps/configs/problem/subselection/multiobjective/test/subset_hpobench_MO_tab_ml_svm_7592.yaml @@ -0,0 +1,37 @@ +# @package _global_ +benchmark_id: HPOBench +problem_id: multiobjective/10/test/hpobench/MO/tab/ml/svm/7592 +problem: + _target_: carps.benchmarks.hpo_bench.HPOBenchProblem + model: svm + task_id: 7592 + metric: + - function_value + - cost + budget_type: null + seed: ${seed} +task: + n_trials: 77 + time_budget: null + n_workers: 1 + n_objectives: 2 + objectives: + - function_value + - cost + is_multifidelity: false + fidelity_type: null + min_budget: null + max_budget: null + has_constraints: false + domain: ML + objective_function_approximation: tabular + has_virtual_time: false + deterministic: false + dimensions: 2 + search_space_n_categoricals: 0 + search_space_n_ordinals: 2 + search_space_n_integers: 0 + search_space_n_floats: 0 + search_space_has_conditionals: false + search_space_has_forbiddens: false + search_space_has_priors: false diff --git a/carps/configs/problem/subselection/multiobjective/test/subset_hpobench_MO_tab_ml_xgboost_168911.yaml b/carps/configs/problem/subselection/multiobjective/test/subset_hpobench_MO_tab_ml_xgboost_168911.yaml new file mode 100644 index 00000000..21c3cf32 --- /dev/null +++ b/carps/configs/problem/subselection/multiobjective/test/subset_hpobench_MO_tab_ml_xgboost_168911.yaml @@ -0,0 +1,37 @@ +# @package _global_ +benchmark_id: HPOBench +problem_id: multiobjective/10/test/hpobench/MO/tab/ml/xgboost/168911 +problem: + _target_: carps.benchmarks.hpo_bench.HPOBenchProblem + model: xgboost + task_id: 168911 + metric: + - function_value + - cost + budget_type: null + seed: ${seed} +task: + n_trials: 100 + time_budget: null + n_workers: 1 + n_objectives: 2 + objectives: + - function_value + - cost + is_multifidelity: false + fidelity_type: null + min_budget: null + max_budget: null + has_constraints: false + domain: ML + objective_function_approximation: tabular + has_virtual_time: false + deterministic: false + dimensions: 4 + search_space_n_categoricals: 0 + search_space_n_ordinals: 4 + search_space_n_integers: 0 + search_space_n_floats: 0 + search_space_has_conditionals: false + search_space_has_forbiddens: false + search_space_has_priors: false diff --git a/subselection/create_subset_configs.py b/subselection/create_subset_configs.py index 377ae0c8..576a6386 100644 --- a/subselection/create_subset_configs.py +++ b/subselection/create_subset_configs.py @@ -40,6 +40,8 @@ def write_subsets(subset_fn: str, identifier: str): if __name__ == "__main__": - # python subselection/create_subset_configs.py subselection/BB_1/default/subset_30.csv subselection/BB_1/default/subset_complement_subset_30.csv blackbox + # python subselection/create_subset_configs.py subselection/BB_2/default/subset_30.csv subselection/BB_2/default/subset_complement_subset_30.csv blackbox + # python subselection/create_subset_configs.py subselection/MF_1/lognorm/subset_20.csv subselection/MF_1/lognorm/subset_complement_subset_20.csv multifidelity + # python subselection/create_subset_configs.py subselection/MO_0/lognorm/subset_10.csv subselection/MO_0/lognorm/subset_complement_subset_10.csv multiobjective fire.Fire(create_subset_configs)