Merge branch 'develop' of https://github.com/AntaresSimulatorTeam/ant…

…ares-xpansion into develop

CMakeLists.txt

@@ -185,6 +185,7 @@ if (MSVC)
 	list(APPEND CMAKE_FIND_LIBRARY_PREFIXES "" "lib")
 endif ()
 find_package(MPI REQUIRED)
+
 find_package(Boost REQUIRED COMPONENTS mpi serialization)
 include_directories(${Boost_INCLUDE_DIRS})
 
@@ -312,12 +313,28 @@ if (USER_GUIDE_PATH)
 endif()
 
 install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/examples/" 		DESTINATION examples)
-
 if (WIN32)
-  install(DIRECTORY ${CURRENT_RUNTIME_OUTPUT_DIRECTORY}/
-    TYPE BIN
-    USE_SOURCE_PERMISSIONS
-    FILES_MATCHING PATTERN "*.dll")
+	cmake_path(GET MPIEXEC_EXECUTABLE PARENT_PATH MPI_BIN_DIR)
+
+	#Copy associated .so and .dll to build directory
+	FILE(COPY  ${MPI_BIN_DIR}/
+		DESTINATION ${CURRENT_RUNTIME_OUTPUT_DIRECTORY}
+		USE_SOURCE_PERMISSIONS)
+	find_file(msmpi_dll NAMES msmpi.dll REGISTRY_VIEW HOST)
+
+	install(DIRECTORY ${CURRENT_RUNTIME_OUTPUT_DIRECTORY}/
+		TYPE BIN
+		USE_SOURCE_PERMISSIONS
+		FILES_MATCHING PATTERN "*.dll")
+
+  	install(FILES
+		${MPI_msmpi_LIBRARY}
+		${msmpi_dll}
+		DESTINATION bin
+        )
+
+	install(DIRECTORY ${MPI_BIN_DIR}/
+  		TYPE BIN)
 endif()
 
 install(DIRECTORY ${ANTARES_SOLVER_DIR}/

conception/Architecture_decision_records/Portable_Xpansion_Windows.md

@@ -0,0 +1,19 @@
+# [Windows package] Export Mpi alongside Xpansion
+
+
+## Status
+
+Accepted (2024/04/12)
+
+## Context
+
+Xpansion packages are not standalone, some dependencies must be pre-installed on the running machines ex MPI. 
+
+## Decision
+
+The decision was taken to release a portable Xpansion Package on Windows only.
+Such a thing is not desired and not even recommended on Linux dist, as Mpi rely on the host machines configuration.
+
+## Consequences
+
+A ready-to-use Xpansion package on Windows  

src/python/antares_xpansion/benders_driver.py

@@ -13,12 +13,14 @@
 
 
 class BendersDriver:
-    def __init__(self, benders, merge_mps, options_file) -> None:
+
+    def __init__(self, benders, merge_mps, options_file, mpiexec=None) -> None:
 
         self.oversubscribe = False
         self.allow_run_as_root = False
         self.benders = benders
         self.merge_mps = merge_mps
+        self.mpiexec = mpiexec
         self.logger = step_logger(__name__, __class__.__name__)
 
         if (options_file != ""):
@@ -132,7 +134,7 @@ def get_mpi_run_command_root(self):
 
     def _initialise_system_specific_mpi_vars(self):
         if sys.platform.startswith("win32"):
-            self.MPI_LAUNCHER = "mpiexec"
+            self.MPI_LAUNCHER = self.mpiexec
         elif sys.platform.startswith("linux"):
             self.MPI_LAUNCHER = "mpirun"
         else:

src/python/antares_xpansion/config_file_parser.py

@@ -1,4 +1,3 @@
-
 import yaml
 from antares_xpansion.xpansionConfig import ConfigParameters
 
@@ -17,6 +16,7 @@ def __init__(self, config_file) -> None:
         self.FULL_RUN_DEFAULT = "full_run"
         self.ANTARES_ARCHIVE_UPDATER_DEFAULT = "antares_archive_updater"
         self.SENSITIVITY_DEFAULT = "sensitivity"
+        self.MPIEXEC_DEFAULT = "mpiexec"
         self.AVAILABLE_SOLVERS_DEFAULT = []
 
     def get_config_parameters(self) -> ConfigParameters:
@@ -27,21 +27,21 @@ def get_config_parameters(self) -> ConfigParameters:
 
             self.config = ConfigParameters(
                 default_install_dir=content.get(
-                    "DEFAULT_INSTALL_DIR", self.default_install_dir),
-                ANTARES=content.get('ANTARES', self.ANTARES_DEFAULT),
-                MERGE_MPS=content.get('MERGE_MPS', self.MERGE_MPS_DEFAULT),
-                BENDERS=content.get(
-                    'BENDERS', self.BENDERS_DEFAULT),
-                LP_NAMER=content.get('LP_NAMER', self.LP_NAMER_DEFAULT),
-                STUDY_UPDATER=content.get(
-                    'STUDY_UPDATER', self.STUDY_UPDATER_DEFAULT),
-                FULL_RUN=content.get(
-                    'FULL_RUN', self.FULL_RUN_DEFAULT),
+                    "DEFAULT_INSTALL_DIR", self.default_install_dir
+                ),
+                ANTARES=content.get("ANTARES", self.ANTARES_DEFAULT),
+                MERGE_MPS=content.get("MERGE_MPS", self.MERGE_MPS_DEFAULT),
+                BENDERS=content.get("BENDERS", self.BENDERS_DEFAULT),
+                LP_NAMER=content.get("LP_NAMER", self.LP_NAMER_DEFAULT),
+                STUDY_UPDATER=content.get("STUDY_UPDATER", self.STUDY_UPDATER_DEFAULT),
+                FULL_RUN=content.get("FULL_RUN", self.FULL_RUN_DEFAULT),
                 ANTARES_ARCHIVE_UPDATER=content.get(
-                    'ANTARES_ARCHIVE_UPDATER', self.ANTARES_ARCHIVE_UPDATER_DEFAULT),
-                SENSITIVITY_EXE=content.get(
-                    'SENSITIVITY', self.SENSITIVITY_DEFAULT),
+                    "ANTARES_ARCHIVE_UPDATER", self.ANTARES_ARCHIVE_UPDATER_DEFAULT
+                ),
+                SENSITIVITY_EXE=content.get("SENSITIVITY", self.SENSITIVITY_DEFAULT),
+                MPIEXEC=content.get("mpiexec", self.MPIEXEC_DEFAULT),
                 AVAILABLE_SOLVERS=content.get(
-                    'AVAILABLE_SOLVER', self.AVAILABLE_SOLVERS_DEFAULT)
+                    "AVAILABLE_SOLVER", self.AVAILABLE_SOLVERS_DEFAULT
+                ),
             )
         return self.config

src/python/antares_xpansion/config_loader.py

@@ -333,7 +333,6 @@ def get_batch_size(self):
 
         return int(batch_size_str)
 
-
     def additional_constraints(self):
         """
         returns path to additional constraints file
@@ -507,38 +506,41 @@ def _set_last_simulation_name(self):
         self._set_xpansion_simulation_name()
     class NotAnXpansionOutputDir(Exception):
         pass
-
 
     def _set_xpansion_simulation_name(self):
         xpansion_dir_suffix ="-Xpansion"
+        self._xpansion_simulation_name = self._last_study
+
         if self.step() in ["resume", "sensitivity"] : 
             self._xpansion_simulation_name = self._last_study
             if self.is_zip(self._last_study):
                 self._xpansion_simulation_name = self._last_study.parent / self._last_study.stem
                 with zipfile.ZipFile(self._last_study, 'r') as output_zip:
                     output_zip.extractall(self._xpansion_simulation_name)
         elif self.step() == "benders":
-            if(not self._last_study.name.endswith(xpansion_dir_suffix)):
-                raise ConfigLoader.NotAnXpansionOutputDir(f"Error! {self._last_study} is not an Xpansion output directory")
-            self._xpansion_simulation_name = self._last_study
+            if self.is_zip(self._last_study):
+                raise ConfigLoader.NotAnXpansionOutputDir(
+                    f"Error! {self._last_study} is not an Xpansion output directory"
+                )
 
         elif self.step() == "problem_generation":
             if not self.is_zip(self._last_study):
                 if(not self._last_study.name.endswith(xpansion_dir_suffix)):
                     raise ConfigLoader.NotAnXpansionOutputDir(f"Error! {self._last_study} is not an Xpansion output directory")
                 else:
                     self._xpansion_simulation_name = self._last_study
-                    self._last_study =self._last_study.parent /  (self._last_study.stem[:-len(xpansion_dir_suffix)]+".zip")
+                    self._last_study = self._last_study.parent / (
+                        self._last_study.stem[: -len(xpansion_dir_suffix)] + ".zip"
+                    )
 
-
         else:
             self._xpansion_simulation_name = self._last_study.parent / \
                 (self._last_study.stem+"-Xpansion")
 
     def is_zip(self, study):
         _, ext = os.path.splitext(study)
         return ext == ".zip" 
-    
+
     def update_last_study_with_sensitivity_results(self):
         if self.is_zip(self._last_study):
             os.remove(self._last_study)
@@ -548,7 +550,7 @@ def update_last_study_with_sensitivity_results(self):
 
     def is_antares_study_output(self, study: Path):
         _, ext = os.path.splitext(study)
-        return ext == ".zip" or (os.path.isdir(study) and '-Xpansion' in study.name)
+        return ext == ".zip" or os.path.isdir(study)
 
     def last_modified_study(self, root_dir:Path)-> Path: 
         list_dir = os.listdir(root_dir)
@@ -563,7 +565,7 @@ def last_modified_study(self, root_dir:Path)-> Path:
         )
         if len(sort_studies) == 0:
             raise ConfigLoader.MissingAntaresOutput("No Antares output is found")
-        
+
         last_study = Path(root_dir) / sort_studies[-1]
         return last_study
 
@@ -609,7 +611,6 @@ def lp_namer_exe(self):
     def benders_exe(self):
         return self.exe_path(self._config.BENDERS)
 
-
     def merge_mps_exe(self):
         return self.exe_path(self._config.MERGE_MPS)
 
@@ -720,3 +721,6 @@ class MissingAntaresOutput(Exception):
     def check_NTC_column_constraints(self, antares_version):
         checker = ChronicleChecker(self._config.data_dir, antares_version)
         checker.check_chronicle_constraints()
+
+    def mpi_exe(self):
+        return self.exe_path(Path(self._config.MPIEXEC).name)

src/python/antares_xpansion/driver.py

@@ -48,7 +48,8 @@ def __init__(self, config_loader: ConfigLoader):
         self.benders_driver = BendersDriver(
             self.config_loader.benders_exe(),
             self.config_loader.merge_mps_exe(),
-            self.config_loader.options_file_name()
+            self.config_loader.options_file_name(),
+            self.config_loader.mpi_exe(),
         )
 
         self.study_update_driver = StudyUpdaterDriver(

src/python/antares_xpansion/full_run_driver.py

@@ -64,8 +64,6 @@ def run(self):
 
         os.chdir(lp_path)
         self.logger.info(f"Current directory is now: {os.getcwd()}")
-        self.logger.info(f"Command is {self.full_command()}")
-        print(self.full_command())
         ret = subprocess.run(
             self.full_command(), shell=False, stdout=sys.stdout, stderr=sys.stderr,
             encoding='utf-8')

src/python/antares_xpansion/xpansionConfig.py

@@ -21,6 +21,7 @@ class ConfigParameters:
     SENSITIVITY_EXE: str
     FULL_RUN: str
     ANTARES_ARCHIVE_UPDATER: str
+    MPIEXEC: str
     AVAILABLE_SOLVERS: List[str]
 
 
@@ -62,6 +63,7 @@ def __init__(
         self.ANTARES_ARCHIVE_UPDATER: str = ""
         self.MPI_LAUNCHER: str = ""
         self.MPI_N: str = ""
+        self.MPIEXEC: str = ""
         self.AVAILABLE_SOLVER: List[str]
 
         self._get_config_values()
@@ -242,4 +244,5 @@ def _get_config_values(self):
         self.FULL_RUN = self.config_parameters.FULL_RUN
         self.ANTARES_ARCHIVE_UPDATER = self.config_parameters.ANTARES_ARCHIVE_UPDATER
         self.SENSITIVITY_EXE = self.config_parameters.SENSITIVITY_EXE
+        self.MPIEXEC = self.config_parameters.MPIEXEC
         self.AVAILABLE_SOLVER = self.config_parameters.AVAILABLE_SOLVERS

src/python/config.yaml.in

@@ -7,5 +7,6 @@ STUDY_UPDATER                   : $<TARGET_FILE_NAME:xpansion-study-updater>
 FULL_RUN                        : $<TARGET_FILE_NAME:full_run>
 SENSITIVITY                     : $<TARGET_FILE_NAME:sensitivity>
 ANTARES_ARCHIVE_UPDATER         : $<TARGET_FILE_NAME:antares_archive_updater>
+mpiexec                         : @MPIEXEC_EXECUTABLE@
 AVAILABLE_SOLVER :
 @AVAILABLE_SOLVER_YML_LIST@

tests/python/test_benders_driver.py

@@ -25,24 +25,24 @@ def setup_method(self):
     def test_lp_path(self, tmp_path):
         lp_path = tmp_path / "lp"
         os.mkdir(lp_path)
-        benders_driver = BendersDriver("", "",  self.OPTIONS_JSON)
+        benders_driver = BendersDriver("", "", self.OPTIONS_JSON, self.MPI_LAUNCHER)
         benders_driver.set_simulation_output_path(tmp_path)
         assert benders_driver.get_lp_path() == lp_path
 
     def test_non_existing_output_path(self, tmp_path):
-        benders_driver = BendersDriver("", "",  self.OPTIONS_JSON)
+        benders_driver = BendersDriver("", "", self.OPTIONS_JSON, self.MPI_LAUNCHER)
         with pytest.raises(BendersDriver.BendersOutputPathError):
             benders_driver.launch(tmp_path / "i_dont_exist", "test", False, 13)
 
     def test_empty_output_path(self, tmp_path):
-        benders_driver = BendersDriver("", "",  self.OPTIONS_JSON)
+        benders_driver = BendersDriver("", "", self.OPTIONS_JSON, self.MPI_LAUNCHER)
         with pytest.raises(BendersDriver.BendersLpPathError):
             benders_driver.launch(tmp_path, "")
 
     def test_illegal_method(self, tmp_path):
         lp_path = tmp_path / "lp"
         os.mkdir(lp_path)
-        benders_driver = BendersDriver("", "",  self.OPTIONS_JSON)
+        benders_driver = BendersDriver("", "", self.OPTIONS_JSON, self.MPI_LAUNCHER)
         with pytest.raises(BendersDriver.BendersSolverError):
             benders_driver.launch(tmp_path, "test")
 
@@ -53,7 +53,9 @@ def test_benders_cmd_mpibenders(self, tmp_path):
         exe_path = os.path.normpath(
             os.path.join(my_install_dir, my_benders_mpi))
 
-        benders_driver = BendersDriver(exe_path, "",  self.OPTIONS_JSON)
+        benders_driver = BendersDriver(
+            exe_path, "", self.OPTIONS_JSON, self.MPI_LAUNCHER
+        )
 
         simulation_output_path = tmp_path
         lp_path = Path(os.path.normpath(
@@ -78,7 +80,8 @@ def test_benders_cmd_mpibenders_with_oversubscribe_linux_only(self, tmp_path):
                 os.path.join(my_install_dir, my_benders_mpi))
 
             benders_driver = BendersDriver(
-                exe_path, "",  self.OPTIONS_JSON)
+                exe_path, "", self.OPTIONS_JSON, self.MPI_LAUNCHER
+            )
 
             simulation_output_path = tmp_path
             lp_path = Path(os.path.normpath(
@@ -100,7 +103,9 @@ def test_benders_cmd_sequential(self, tmp_path):
         exe_path = os.path.normpath(
             os.path.join(my_install_dir, my_sequential))
 
-        benders_driver = BendersDriver(exe_path, "",  self.OPTIONS_JSON)
+        benders_driver = BendersDriver(
+            exe_path, "", self.OPTIONS_JSON, self.MPI_LAUNCHER
+        )
 
         simulation_output_path = tmp_path
         lp_path = Path(os.path.normpath(
@@ -120,7 +125,9 @@ def test_benders_cmd_merge_mps(self, tmp_path):
         exe_path = os.path.normpath(
             os.path.join(my_install_dir, my_merges_mps))
 
-        benders_driver = BendersDriver("",  exe_path, self.OPTIONS_JSON)
+        benders_driver = BendersDriver(
+            "", exe_path, self.OPTIONS_JSON, self.MPI_LAUNCHER
+        )
 
         simulation_output_path = tmp_path
         lp_path = Path(os.path.normpath(
@@ -142,7 +149,9 @@ def test_raise_execution_error(self, tmp_path):
         exe_path = os.path.normpath(
             os.path.join(my_install_dir, my_benders_mpi))
 
-        benders_driver = BendersDriver(exe_path, "",  self.OPTIONS_JSON)
+        benders_driver = BendersDriver(
+            exe_path, "", self.OPTIONS_JSON, self.MPI_LAUNCHER
+        )
 
         simulation_output_path = tmp_path
         lp_path = Path(os.path.normpath(
@@ -161,7 +170,9 @@ def test_clean_solver_log_file(self, tmp_path):
         my_n_mpi = 13
         exe_path = os.path.normpath(
             os.path.join(my_install_dir, my_benders_mpi))
-        benders_driver = BendersDriver(exe_path, "",  self.OPTIONS_JSON)
+        benders_driver = BendersDriver(
+            exe_path, "", self.OPTIONS_JSON, self.MPI_LAUNCHER
+        )
 
         simulation_output_path = tmp_path
         lp_path = Path(os.path.normpath(
@@ -191,7 +202,7 @@ def test_unsupported_platform(self, tmp_path):
         with patch(MOCK_SYS, autospec=True) as sys_:
             sys_.platform = "exotic_platform"
             with pytest.raises(BendersDriver.BendersUnsupportedPlatform):
-                BendersDriver("",  "", self.OPTIONS_JSON)
+                BendersDriver("", "", self.OPTIONS_JSON, self.MPI_LAUNCHER)
 
     def test_clean_benders_step_if_not_keep_mps(self, tmp_path):
         my_benders_mpi = "something"
@@ -200,7 +211,9 @@ def test_clean_benders_step_if_not_keep_mps(self, tmp_path):
             os.path.join(my_install_dir, my_benders_mpi))
         keep_mps = False
 
-        benders_driver = BendersDriver(exe_path, "",  self.OPTIONS_JSON)
+        benders_driver = BendersDriver(
+            exe_path, "", self.OPTIONS_JSON, self.MPI_LAUNCHER
+        )
 
         simulation_output_path = tmp_path
         lp_path = Path(os.path.normpath(