Create lp directories in cpp (#710)

    Don't delete old log
    Don't delete existing -xpansion directory
    Create -xpansion and lp directories during problem generation CPP
    Remove "dead" code path using results from file or from antares: simplify, only keep zip path

conception/SequenceDiagrams/FileManipulation/diagram.txt

@@ -68,6 +68,8 @@ create ZipProblemProviderAdapter
 create FilesMapper
 full_run->ProblemGeneration:RunProblemGeneration
 activate ProblemGeneration
+ProblemGeneration->ProblemGeneration:CreateDirectories
+note over ProblemGeneration: Create -Xpansion and -Xpansion/lp directories
 ProblemGeneration->ProblemGeneration:ExtractUtilsFiles
 
 ProblemGeneration->LpFilesExtractor:ExtractFiles

src/cpp/exe/lpnamer/main.cpp

@@ -5,6 +5,18 @@
 #include "ProblemGenerationLogger.h"
 #include "RunProblemGeneration.h"
 
+static const std::string LP_DIRNAME = "lp";
+
+void CreateDirectories(const std::filesystem::path& output_path) {
+  if (!std::filesystem::exists(output_path)) {
+    std::filesystem::create_directories(output_path);
+  }
+  auto lp_path = output_path / LP_DIRNAME;
+  if (!std::filesystem::exists(lp_path)) {
+    std::filesystem::create_directories(lp_path);
+  }
+}
+
 int main(int argc, char** argv) {
   try {
     auto options_parser = ProblemGenerationExeOptions();
@@ -16,9 +28,9 @@ int main(int argc, char** argv) {
     const auto log_file_path =
         xpansion_output_dir / "lp" / "ProblemGenerationLog.txt";
 
+    CreateDirectories(xpansion_output_dir);
     auto logger = ProblemGenerationLog::BuildLogger(log_file_path, std::cout,
                                                     "Problem Generation");
-    auto& loggerRef = (*logger);
 
     auto master_formulation = options_parser.MasterFormulation();
     auto additionalConstraintFilename_l =

src/cpp/lpnamer/helper/ProblemGenerationLogger.cpp

@@ -7,10 +7,11 @@ ProblemGenerationFileLogger::ProblemGenerationFileLogger(
     const std::filesystem::path& logFilePath)
     : logFilePath_(logFilePath) {
   SetType(LogUtils::LOGGERTYPE::FILE);
-  logFile_.open(logFilePath_, std::ofstream::out | std::ofstream::app);
+  auto logPath = std::filesystem::absolute(logFilePath_);
+  logFile_.open(logPath, std::ofstream::out | std::ofstream::app);
   if (logFile_.fail()) {
     std::cerr << "ProblemGenerationFileLogger: Invalid file ("
-              << logFilePath_.string() << ") passed as parameter" << std::endl;
+              << logPath.string() << ") passed as parameter" << std::endl;
   }
 }
 void ProblemGenerationFileLogger::DisplayMessage(const std::string& message) {

src/cpp/lpnamer/main/RunProblemGeneration.cpp

@@ -1,13 +1,11 @@
 
 #include "RunProblemGeneration.h"
 
-#include <boost/archive/text_iarchive.hpp>
 #include <execution>
 #include <iostream>
 
 #include "ActiveLinks.h"
 #include "AdditionalConstraints.h"
-#include "Clock.h"
 #include "GeneralDataReader.h"
 #include "LauncherHelpers.h"
 #include "LinkProblemsGenerator.h"
@@ -17,14 +15,12 @@
 #include "MasterGeneration.h"
 #include "MasterProblemBuilder.h"
 #include "MpsTxtWriter.h"
-#include "ProblemGenerationExeOptions.h"
 #include "ProblemVariablesFromProblemAdapter.h"
 #include "ProblemVariablesZipAdapter.h"
 #include "StringManip.h"
 #include "Timer.h"
 #include "WeightsFileReader.h"
 #include "WeightsFileWriter.h"
-#include "XpansionProblemsFromAntaresProvider.h"
 #include "ZipProblemsProviderAdapter.h"
 #include "config.h"
 
@@ -59,6 +55,7 @@ struct Version {
   int major;
   int minor;
 };
+
 std::shared_ptr<ArchiveReader> InstantiateZipReader(
     const std::filesystem::path& antares_archive_path);
 void ProcessWeights(
@@ -128,6 +125,7 @@ std::vector<std::shared_ptr<Problem>> getXpansionProblems(
                                                               problem_names);
   return adapter->provideProblems(solver_name, solver_log_manager);
 }
+
 void RunProblemGeneration(
     const std::filesystem::path& xpansion_output_dir,
     const std::string& master_formulation,
@@ -169,14 +167,10 @@ void RunProblemGeneration(
   auto files_mapper = FilesMapper(antares_archive_path);
   auto mpsList = files_mapper.MpsAndVariablesFilesVect();
 
-  bool use_zip_implementation = true;
-  bool use_file_implementation = false;
-
   auto solver_log_manager = SolverLogManager(log_file_path);
   Couplings couplings;
   LinkProblemsGenerator linkProblemsGenerator(
       lpDir_, links, solver_name, logger, solver_log_manager, rename_problems);
-  if (use_zip_implementation) {
     std::shared_ptr<ArchiveReader> reader =
         InstantiateZipReader(antares_archive_path);
 
@@ -185,79 +179,33 @@ void RunProblemGeneration(
         getXpansionProblems(solver_log_manager, solver_name, mpsList, lpDir_,
                             reader);
 
-    std::vector<std::pair<std::shared_ptr<Problem>, ProblemData>>
-        problems_and_data;
-    for (int i = 0; i < xpansion_problems.size(); ++i) {
-      xpansion_problems.at(i)->_name = mpsList.at(i)._problem_mps;
-      problems_and_data.emplace_back(xpansion_problems.at(i), mpsList.at(i));
-    }
-    auto mps_file_writer = std::make_shared<MPSFileWriter>(lpDir_);
-    std::for_each(
-        std::execution::par, problems_and_data.begin(), problems_and_data.end(),
-        [&](const auto& problem_and_data) {
-          const auto& [problem, data] = problem_and_data;
-          std::shared_ptr<IProblemVariablesProviderPort> variables_provider;
-          if (rename_problems) {
-            variables_provider = std::make_shared<ProblemVariablesZipAdapter>(
-                reader, data, links, logger);
-          } else {
-            variables_provider =
-                std::make_shared<ProblemVariablesFromProblemAdapter>(
-                    problem, links, logger);
-          }
-          linkProblemsGenerator.treat(data._problem_mps, couplings, problem.get(),
-                                      variables_provider.get(), mps_file_writer.get());
-        });
-
-    reader->Close();
-    reader->Delete();
-  } else if (use_file_implementation) {
-    /* Main stuff */
-    auto mps_file_writer = std::make_shared<MPSFileWriter>(lpDir_);
-    linkProblemsGenerator.treatloop(xpansion_output_dir, couplings, mpsList,
+  std::vector<std::pair<std::shared_ptr<Problem>, ProblemData>>
+      problems_and_data;
+  for (int i = 0; i < xpansion_problems.size(); ++i) {
+    xpansion_problems.at(i)->_name = mpsList.at(i)._problem_mps;
+    problems_and_data.emplace_back(xpansion_problems.at(i), mpsList.at(i));
+  }
+  auto mps_file_writer = std::make_shared<MPSFileWriter>(lpDir_);
+  std::for_each(
+      std::execution::par, problems_and_data.begin(), problems_and_data.end(),
+      [&](const auto& problem_and_data) {
+        const auto& [problem, data] = problem_and_data;
+        std::shared_ptr<IProblemVariablesProviderPort> variables_provider;
+        if (rename_problems) {
+          variables_provider = std::make_shared<ProblemVariablesZipAdapter>(
+              reader, data, links, logger);
+        } else {
+          variables_provider =
+              std::make_shared<ProblemVariablesFromProblemAdapter>(
+                  problem, links, logger);
+        }
+        linkProblemsGenerator.treat(data._problem_mps, couplings, problem.get(),
+                                    variables_provider.get(),
                                     mps_file_writer.get());
-  } else {
-    std::filesystem::path path =
-        xpansion_output_dir.parent_path().parent_path() /
-        "fichierDeSerialisation";
-    std::ifstream ifs(xpansion_output_dir.parent_path().parent_path() /
-                      "fichierDeSerialisation");
-    boost::archive::text_iarchive ia(ifs);
-
-    LpsFromAntares lps;
-    ia >> lps;
-    lps._constant->Mdeb.push_back(lps._constant->NombreDeCoefficients);
-
-    XpansionProblemsFromAntaresProvider adapter(lps);
-    auto xpansion_problems =
-        adapter.provideProblems(solver_name, solver_log_manager);
-    std::vector<std::pair<std::shared_ptr<Problem>, ProblemData>>
-        problems_and_data;
-    for (int i = 0; i < xpansion_problems.size(); ++i) {
-      xpansion_problems.at(i)->_name = mpsList.at(i)._problem_mps;
-      problems_and_data.emplace_back(xpansion_problems.at(i), mpsList.at(i));
-    }
-
-    auto reader = InstantiateZipReader(antares_archive_path);
-    auto mps_file_writer = std::make_shared<MPSFileWriter>(lpDir_);
+      });
 
-    std::for_each(
-        std::execution::par, problems_and_data.begin(), problems_and_data.end(),
-        [&](const auto& problem_and_data) {
-          const auto& [problem, data] = problem_and_data;
-          std::shared_ptr<IProblemVariablesProviderPort> variables_provider;
-          if (rename_problems) {
-            variables_provider = std::make_shared<ProblemVariablesZipAdapter>(
-                reader, data, links, logger);
-          } else {
-            variables_provider =
-                std::make_shared<ProblemVariablesFromProblemAdapter>(
-                    problem, links, logger);
-          }
-          linkProblemsGenerator.treat(data._problem_mps, couplings, problem.get(),
-                                      variables_provider.get(), mps_file_writer.get());
-        });
-  }
+  reader->Close();
+  reader->Delete();
 
   MasterGeneration master_generation(
       xpansion_output_dir, links, additionalConstraints, couplings,

src/python/antares_xpansion/full_run_driver.py

@@ -1,13 +1,13 @@
-
-from typing import List
+import os
 import subprocess
 import sys
+from pathlib import Path
+from typing import List
+
 from antares_xpansion.benders_driver import BendersDriver
-from antares_xpansion.problem_generator_driver import ProblemGeneratorDriver
 from antares_xpansion.logger import step_logger
+from antares_xpansion.problem_generator_driver import ProblemGeneratorDriver
 from antares_xpansion.study_output_cleaner import StudyOutputCleaner
-import os
-from pathlib import Path
 
 
 class FullRunDriver:

src/python/antares_xpansion/problem_generator_driver.py

@@ -80,7 +80,7 @@ def set_output_path(self, output_path):
         if output_path.exists():
             self._output_path = output_path
             self.xpansion_output_dir = output_path.parent / \
-                (output_path.stem+"-Xpansion")
+                                       (output_path.stem + "-Xpansion")
             if self.xpansion_output_dir.exists():
                 shutil.rmtree(self.xpansion_output_dir)
             os.makedirs(self.xpansion_output_dir)

tests/cpp/lp_namer/CMakeLists.txt

@@ -36,7 +36,8 @@ add_executable (lp_namer_tests
         WeightsFileReaderTest.cpp
         LpFilesExtractorTest.cpp
         MpsTxtWriterTest
-        )
+        ProblemGenerationTest.cpp
+)
 
 target_link_libraries (lp_namer_tests PRIVATE
         GTest::Main

tests/cpp/lp_namer/ProblemGenerationTest.cpp

@@ -0,0 +1,32 @@
+//
+// Created by marechaljas on 10/10/23.
+//
+
+#include <filesystem>
+
+#include "ArchiveIO.h"
+#include "LoggerBuilder.h"
+#include "RunProblemGeneration.h"
+#include "gtest/gtest.h"
+
+TEST(InitializationTest, FoldersAreEmpty) {
+  auto workingDirectory =
+      std::filesystem::temp_directory_path() / std::tmpnam(nullptr);
+  auto simulationDirectory = workingDirectory / "output" / "simulation";
+  auto logger = emptyLogger();
+  std::filesystem::create_directories(simulationDirectory);
+
+  auto xpansionDirectory =
+      (simulationDirectory.parent_path() /
+       (simulationDirectory.filename().string() + "-Xpansion"));
+  std::filesystem::create_directories(xpansionDirectory);
+  std::ofstream emptyfile(xpansionDirectory / "garbage.txt");
+
+  EXPECT_TRUE(std::filesystem::exists(xpansionDirectory / "garbage.txt"));
+  EXPECT_THROW(
+      RunProblemGeneration(simulationDirectory, "integer", "", "", logger,
+                           simulationDirectory / "logs.txt", "", false),
+      ArchiveIOGeneralException);
+
+  EXPECT_FALSE(std::filesystem::exists(simulationDirectory / "garbage.txt"));
+}

tests/python/test_problem_generator_driver.py

@@ -1,16 +1,10 @@
-from typing import Text
-import pytest
-import os
-from pathlib import Path
-from datetime import date, datetime
-from unittest.mock import ANY, mock_open, patch
 import shutil
+from datetime import date, datetime
+from pathlib import Path
+from unittest.mock import patch
 
-from file_creation import _create_weight_file
+import pytest
 from antares_xpansion.problem_generator_driver import ProblemGeneratorData, ProblemGeneratorDriver
-from antares_xpansion.antares_driver import AntaresDriver
-from antares_xpansion.xpansion_study_reader import XpansionStudyReader
-from antares_xpansion.general_data_reader import GeneralDataIniReader
 
 SUBPROCESS_RUN = "antares_xpansion.problem_generator_driver.subprocess.run"
 zipfile_ZipFile = "antares_xpansion.problem_generator_driver.zipfile.ZipFile"