abstract intensifier has a FIXME check. Maybe this is cause the problem but a config should be added to rejected configs if the incumbent set stays the same.

Author: mwever

smac/intensifier/abstract_intensifier.py

@@ -627,8 +627,8 @@ def update_incumbents(self, config: Configuration) -> None:
 
         # Update trajectory
         if previous_incumbents == new_incumbents:  # Only happens with incumbent config
-            self._remove_rejected_config(config)  # Remove the incumbent from the rejected config list
-
+            # self._remove_rejected_config(config)  # Remove the incumbent from the rejected config list
+            self._add_rejected_config(config)  # FIXME check whether this is really what we want
             return
         elif len(previous_incumbents) == len(new_incumbents):
             # In this case, we have to determine which config replaced which incumbent and reject it

smac/intensifier/intensifier.py

@@ -340,7 +340,7 @@ def __iter__(self) -> Iterator[TrialInfo]:
 
     def _check_for_intermediate_comparison(self, config: Configuration) -> bool:
         """Checks if the configuration should be evaluated against the incumbent while it
-        did not run on all the trails the incumbents did.By default this triggers when all N trails have completed.
+        did not run on all the trails the incumbents did. By default this triggers when all N trails have completed.
 
 
         Parameters
@@ -354,6 +354,11 @@ def _check_for_intermediate_comparison(self, config: Configuration) -> bool:
         config_isb_keys = self.get_instance_seed_budget_keys(config)
         config_hash = get_config_hash(config)
 
+        if (
+            self.uses_cutoffs and len(config_isb_keys) > 0
+        ):  # if we are in a runtime setting we definitely want to do intermediate comparisons
+            return True
+
         # Do not compare very early in the process
         if len(config_isb_keys) < 4:
             return False

smac/main/config_selector.py

@@ -252,7 +252,7 @@ def __iter__(self) -> Iterator[Configuration]:
             if self._retrain_wallclock_ratio is not None:
                 # TODO: CB: What does this actually do? Delete/clear the iterator?
                 #  --> JG: To easily measure the time needed to perform maximise, difficult otherwise due to yield
-                len(challengers)  # Forces actual computation of the acquisition function maximizer
+                len(list(challengers))  # Forces actual computation of the acquisition function maximizer
 
             self._acquisition_training_times.append(time.time() - train_start_time)
 

tests/test_intensifier/test_adaptive_capping_intensifier.py

@@ -5,6 +5,7 @@
 from ConfigSpace import ConfigurationSpace, Configuration
 
 from smac import Scenario, AlgorithmConfigurationFacade
+from smac.runhistory import TrialInfo, TrialValue
 import numpy as np
 
 from smac.main.exceptions import ConfigurationSpaceExhaustedException
@@ -74,12 +75,11 @@ def train(self, config:Configuration, instance: str, cutoff: int, seed: int = 0)
                 if log[1] == "reject" and log[2] == config_hash:
                     reject_log = log
                     break
-
             self.expected_behavior_violations += [f"{self.log_counter}: Configuration {config_hash} was already "
                                                   f"rejected here {reject_log}"]
 
         # specify log entry (id, config hash, instance, performance)
-        log = (self.log_counter, config_hash, instance, rand_perf)
+        log = (self.log_counter, config_hash, instance, int(rand_perf))
 
         # ensure config has an entry in the log map and store observed performance, add log entry to list
         if config_hash not in self.log_map:
@@ -139,12 +139,15 @@ def test_incumbent_switch() -> None:
 
     # setup test environment
     tm = TrainMockup()
-    smac = get_basic_setup(tm.train)
+    smac = get_basic_setup(tm.train, num_configs=100)
 
     # start smac run
     try:
-        smac.optimize()
+        for i in range(20):
+            trial_info:TrialInfo = smac.ask()
+            val = tm.train(config=trial_info.config, instance=trial_info.instance, cutoff=trial_info.additional_info["cutoff"])
+            smac.tell(trial_info, TrialValue(cost=val))
     except ConfigurationSpaceExhaustedException:
         pass
 
-    assert tm.expected_behavior_violated is False, tm.get_violation_report()
+    assert tm.expected_behavior_violated is False, tm.get_violation_report()