Import modeler data-series

Signed-off-by: Peter Mitri <[email protected]>

src/io/inputs/data-series-csv-importer/CMakeLists.txt

@@ -15,9 +15,9 @@ target_include_directories(data-series-csv-importer
 
 # Link dependencies
 target_link_libraries(data-series-csv-importer
-        PRIVATE
-        linear-problem-data-impl
         PUBLIC
+        linear-problem-data-impl
+        PRIVATE
         Boost::headers
 )
 

src/io/inputs/data-series-csv-importer/DataSeriesRepoImporter.cpp

@@ -23,73 +23,85 @@
 #include <boost/regex.hpp>
 
 #include <antares/io/inputs/data-series-csv-importer/DataSeriesRepoImporter.h>
-#include <antares/optimisation/linear-problem-data-impl/dataSeriesRepo.h>
-#include <antares/optimisation/linear-problem-data-impl/timeSeriesSet.h>
 
 namespace Antares::IO::Inputs::DataSeriesCsvImporter
 {
 using namespace std;
 using namespace boost;
 using namespace Optimisation::LinearProblemDataImpl;
 
-DataSeriesRepository DataSeriesRepoImporter::importFromDirectory(const string& path,
+DataSeriesRepository DataSeriesRepoImporter::importFromDirectory(const std::filesystem::path& path,
                                                                  char csvSeparator)
 {
-    return DataSeriesRepository();
+    if (!is_directory(path))
+    {
+        throw invalid_argument("Not a directory: " + path.string());
+    }
+    DataSeriesRepository repo{};
+    for (const auto& entry: std::filesystem::directory_iterator(path))
+    {
+        if (!is_regular_file(entry))
+        {
+            continue;
+        }
+        if (entry.path().extension() == ".csv")
+        {
+            std::unique_ptr<IDataSeries> timeSeriesSet = make_unique<TimeSeriesSet>(
+              TimeSeriesSetImporter::importFromFile(entry, csvSeparator));
+            repo.addDataSeries(std::move(timeSeriesSet));
+        }
+    }
+    return repo;
 }
 
-TimeSeriesSet TimeSeriesSetImporter::importFromFile(const string& path, char csvSeparator)
+vector<vector<double>> TimeSeriesSetImporter::csvToMatrix(const std::filesystem::path& path,
+                                                          char csvSeparator)
 {
-    // used to split the file in lines
-    const regex linesregx("\\r\\n|\\n\\r|\\n|\\r");
-    // used to split each line to tokens, assuming ',' as column separator
-    const regex fieldsregx(csvSeparator + "(?=(?:[^\"]*\"[^\"]*\")*(?![^\"]*\"))");
+    // TODO: add logs?
+    // logs.debug() << "Loading data-series from " << path;
     vector<vector<double>> result;
-    ifstream infile;
-    infile.open(path);
-    char data[1024];
-    infile.read(data, sizeof(data));
-    data[infile.tellg()] = '\0';
-    unsigned int length = strlen(data);
-
-    // iterator splits data to lines
-    cregex_token_iterator li(data, data + length, linesregx, -1);
-    cregex_token_iterator end;
-
-    while (li != end)
+    ifstream infile(path, std::ios_base::binary | std::ios_base::in);
+    if (!infile.is_open())
+    {
+        throw invalid_argument("Could not open file " + path.filename().string());
+    }
+    string line;
+    string element;
+    while (!infile.eof())
     {
-        string line = li->str();
-        ++li;
-
-        // Split line to tokens
-        sregex_token_iterator ti(line.begin(), line.end(), fieldsregx, -1);
-        sregex_token_iterator end2;
-
         vector<double> row;
-        while (ti != end2)
+        getline(infile, line);
+        std::stringstream ss(line);
+        while (getline(ss, element, csvSeparator))
         {
-            double token = stod(ti->str());
-            ++ti;
-            row.push_back(token);
+            row.push_back(stod(element));
         }
         result.push_back(row);
     }
+    return move(result);
+}
+
+TimeSeriesSet TimeSeriesSetImporter::importFromFile(const std::filesystem::path& path,
+                                                    char csvSeparator)
+{
+    auto csvMatrix = csvToMatrix(path, csvSeparator);
     // We have to transpose the matrix
-    string id = path; // TODO
-    if (result.empty())
+    // TODO: we may want to improve this by reading directly into the TimeSeriesSet object, or
+    // by creating a specific IDataSeries implementation
+    int nTimestamps = csvMatrix.size();
+    TimeSeriesSet timeSeriesSet(path.stem(), nTimestamps);
+    if (nTimestamps == 0)
     {
-        return TimeSeriesSet(id, 0);
+        return timeSeriesSet;
     }
-    int nTimestamps = result.size();
-    TimeSeriesSet timeSeriesSet(id, nTimestamps);
-    int nSets = result[0].size();
+    int nSets = csvMatrix[0].size();
     for (int i = 0; i < nSets; ++i)
     {
         vector<double> set;
         set.reserve(nTimestamps);
         for (int j = 0; j < nTimestamps; ++j)
         {
-            set.push_back(result[j][i]);
+            set.push_back(csvMatrix[j][i]);
         }
         timeSeriesSet.add(set);
     }

src/io/inputs/data-series-csv-importer/include/antares/io/inputs/data-series-csv-importer/DataSeriesRepoImporter.h

@@ -20,13 +20,10 @@
 */
 #pragma once
 
-#include <string>
+#include <filesystem>
 
-namespace Antares::Optimisation::LinearProblemDataImpl
-{
-class DataSeriesRepository;
-class TimeSeriesSet;
-} // namespace Antares::Optimisation::LinearProblemDataImpl
+#include <antares/optimisation/linear-problem-data-impl/dataSeriesRepo.h>
+#include <antares/optimisation/linear-problem-data-impl/timeSeriesSet.h>
 
 /**
  * Reads a DataSeriesRepo from a directory
@@ -38,16 +35,22 @@ namespace Antares::IO::Inputs::DataSeriesCsvImporter
 class TimeSeriesSetImporter
 {
 public:
+    TimeSeriesSetImporter() = delete; // must not be used
     static Optimisation::LinearProblemDataImpl::TimeSeriesSet importFromFile(
-      const std::string& path,
+      const std::filesystem::path& path,
       char csvSeparator = ';');
+
+private:
+    static std::vector<std::vector<double>> csvToMatrix(const std::filesystem::path& path,
+                                                        char csvSeparator = ';');
 };
 
 class DataSeriesRepoImporter
 {
 public:
+    DataSeriesRepoImporter() = delete; // must not be used
     static Optimisation::LinearProblemDataImpl::DataSeriesRepository importFromDirectory(
-      const std::string& path,
+      const std::filesystem::path&,
       char csvSeparator = ';');
 };
 

src/optimisation/linear-problem-data-impl/include/antares/optimisation/linear-problem-data-impl/linearProblemData.h

@@ -34,6 +34,13 @@ namespace Antares::Optimisation::LinearProblemDataImpl
 class LinearProblemData: public LinearProblemApi::ILinearProblemData
 {
 public:
+    LinearProblemData() = default;
+
+    LinearProblemData(DataSeriesRepository& data_series_repository):
+        dataSeriesRepository_(std::move(data_series_repository))
+    {
+    }
+
     double getData(const std::string& dataSetId,
                    const std::string& scenarioGroup,
                    const unsigned scenario,

src/optimisation/linear-problem-data-impl/linearProblemData.cpp

@@ -40,7 +40,9 @@ double LinearProblemData::getData(const std::string& dataSetId,
                                   const unsigned scenario,
                                   const unsigned hour)
 {
-    unsigned rank = groupRepository_.getDataRank(scenarioGroup, scenario);
+    // TODO: use the following line to add scenario group support
+    // unsigned rank = groupRepository_.getDataRank(scenarioGroup, scenario);
+    unsigned rank = scenario;
     return dataSeriesRepository_.getDataSeries(dataSetId).getData(rank, hour);
 }
 

src/solver/modeler/CMakeLists.txt

@@ -24,6 +24,7 @@ target_link_libraries(modeler-lib
         Antares::linear-problem-api
         # TODO FIXME don't depend on implementations
         Antares::linear-problem-mpsolver-impl
+        Antares::data-series-csv-importer
 )
 
 target_link_libraries(antares-modeler
@@ -32,6 +33,7 @@ target_link_libraries(antares-modeler
         linear-problem-data-impl
         linear-problem-mpsolver-impl
         optim-model-filler
+        data-series-csv-importer
 )
 
 import_std_libs(antares-modeler)

src/solver/modeler/main.cpp

@@ -21,19 +21,21 @@
 
 #include <fstream>
 
+#include <antares/io/inputs/data-series-csv-importer/DataSeriesRepoImporter.h>
 #include <antares/logs/logs.h>
+#include <antares/optimisation/linear-problem-api/linearProblem.h>
 #include <antares/optimisation/linear-problem-api/linearProblemBuilder.h>
 #include <antares/optimisation/linear-problem-data-impl/linearProblemData.h>
 #include <antares/optimisation/linear-problem-mpsolver-impl/linearProblem.h>
 #include <antares/solver/modeler/loadFiles/loadFiles.h>
 #include <antares/solver/modeler/parameters/parseModelerParameters.h>
 #include <antares/solver/optim-model-filler/ComponentFiller.h>
-#include "antares/optimisation/linear-problem-api/linearProblem.h"
 
 using namespace Antares::Optimisation::LinearProblemMpsolverImpl;
 using namespace Antares;
 using namespace Antares::Solver;
 using namespace Antares::Optimisation::LinearProblemApi;
+using namespace Antares::Optimisation::LinearProblemDataImpl;
 
 class SystemLinearProblem
 {
@@ -45,7 +47,9 @@ class SystemLinearProblem
 
     ~SystemLinearProblem() = default;
 
-    void Provide(ILinearProblem& pb, const ModelerParameters& parameters)
+    void Provide(ILinearProblem& pb,
+                 const ModelerParameters& parameters,
+                 DataSeriesRepository& dataSeriesRepo)
     {
         std::vector<std::unique_ptr<Optimization::ComponentFiller>> fillers;
         std::vector<LinearProblemFiller*> fillers_ptr;
@@ -60,9 +64,9 @@ class SystemLinearProblem
         }
 
         LinearProblemBuilder linear_problem_builder(fillers_ptr);
-        Optimisation::LinearProblemDataImpl::LinearProblemData dummy_data;
+        LinearProblemData data(dataSeriesRepo);
         FillContext dummy_time_scenario_ctx = {parameters.firstTimeStep, parameters.lastTimeStep};
-        linear_problem_builder.build(pb, dummy_data, dummy_time_scenario_ctx);
+        linear_problem_builder.build(pb, data, dummy_time_scenario_ctx);
     }
 
 private:
@@ -102,6 +106,18 @@ int main(int argc, const char** argv)
         logs.info() << "Libraries loaded";
         const auto system = LoadFiles::loadSystem(studyPath, libraries);
         logs.info() << "System loaded";
+        DataSeriesRepository dataSeriesRepository;
+        try
+        {
+            dataSeriesRepository = IO::Inputs::DataSeriesCsvImporter::DataSeriesRepoImporter::
+              importFromDirectory(studyPath / "input" / "data-series", ';');
+            logs.info() << "Data-series loaded";
+        }
+        catch (const std::exception& e)
+        {
+            // Only warning, because data-series are not mandatory
+            logs.warning() << "Data series could not be imported: " << e.what();
+        }
         SystemLinearProblem system_linear_problem(system);
 
         auto outputPath = studyPath / "output";
@@ -119,7 +135,7 @@ int main(int argc, const char** argv)
         logs.info() << "linear problem of System loaded";
         OrtoolsLinearProblem ortools_linear_problem(true, parameters.solver);
 
-        system_linear_problem.Provide(ortools_linear_problem, parameters);
+        system_linear_problem.Provide(ortools_linear_problem, parameters, dataSeriesRepository);
 
         logs.info() << "Linear problem provided";
 

src/tests/src/CMakeLists.txt

@@ -9,4 +9,5 @@ add_subdirectory(solver)
 add_subdirectory(expressions)
 add_subdirectory(optimisation)
 add_subdirectory(study)
+add_subdirectory(io)
 add_subdirectory(modeler)

src/tests/src/io/data-series-csv-importer/CMakeLists.txt

@@ -0,0 +1,10 @@
+include(${CMAKE_SOURCE_DIR}/tests/macros.cmake)
+
+add_boost_test(TestDataSeriesCsvImporter
+  SRC
+  test_main.cpp
+  testDataSeriesCsvImporter.cpp
+  LIBS
+  Antares::data-series-csv-importer
+  Boost::unit_test_framework
+  test_utils_unit)