diff --git a/src/io/inputs/data-series-csv-importer/DataSeriesRepoImporter.cpp b/src/io/inputs/data-series-csv-importer/DataSeriesRepoImporter.cpp index 27b641931e..7d7f8cf2d7 100644 --- a/src/io/inputs/data-series-csv-importer/DataSeriesRepoImporter.cpp +++ b/src/io/inputs/data-series-csv-importer/DataSeriesRepoImporter.cpp @@ -26,8 +26,6 @@ namespace Antares::IO::Inputs::DataSeriesCsvImporter { -using namespace std; -using namespace boost; using namespace Optimisation::LinearProblemDataImpl; DataSeriesRepository DataSeriesRepoImporter::importFromDirectory(const std::filesystem::path& path, @@ -35,7 +33,7 @@ DataSeriesRepository DataSeriesRepoImporter::importFromDirectory(const std::file { if (!is_directory(path)) { - throw invalid_argument("Not a directory: " + path.string()); + throw std::invalid_argument("Not a directory: " + path.string()); } DataSeriesRepository repo{}; for (const auto& entry: std::filesystem::directory_iterator(path)) @@ -44,9 +42,9 @@ DataSeriesRepository DataSeriesRepoImporter::importFromDirectory(const std::file { continue; } - if (entry.path().extension() == ".csv") + if (entry.path().extension() == ".csv" || entry.path().extension() == ".tsv") { - std::unique_ptr timeSeriesSet = make_unique( + std::unique_ptr timeSeriesSet = std::make_unique( TimeSeriesSetImporter::importFromFile(entry, csvSeparator)); repo.addDataSeries(std::move(timeSeriesSet)); } @@ -54,31 +52,60 @@ DataSeriesRepository DataSeriesRepoImporter::importFromDirectory(const std::file return repo; } -vector> TimeSeriesSetImporter::csvToMatrix(const std::filesystem::path& path, - char csvSeparator) +std::vector> TimeSeriesSetImporter::csvToMatrix( + const std::filesystem::path& path, + char csvSeparator) { - // TODO: add logs? - // logs.debug() << "Loading data-series from " << path; - vector> result; - ifstream infile(path, std::ios_base::binary | std::ios_base::in); + std::vector> result; + std::ifstream infile(path, std::ios_base::binary | std::ios_base::in); if (!infile.is_open()) { - throw invalid_argument("Could not open file " + path.filename().string()); + throw std::invalid_argument("Could not open file " + path.filename().string()); } - string line; - string element; + std::string line; + std::string element; + bool empty_line_found = false; while (!infile.eof()) { - vector row; + std::vector row; getline(infile, line); + if (line.empty()) + { + empty_line_found = true; + continue; + } + if (empty_line_found) + { + // only accept empty lines in the end of the file + throw std::invalid_argument(path.filename().string() + + ": empty line in the middle of the file"); + } std::stringstream ss(line); while (getline(ss, element, csvSeparator)) { - row.push_back(stod(element)); + if (element.empty()) + { + throw std::invalid_argument(path.filename().string() + + ": columns have inconsistent number of rows"); + } + try + { + row.push_back(std::stod(element)); + } + catch (const std::invalid_argument& e) + { + throw std::invalid_argument(path.filename().string() + ": \"" + element + + "\" is not a number"); + } + } + if (!result.empty() && row.size() != result[0].size()) + { + throw std::invalid_argument(path.filename().string() + + ": rows have inconsistent number of columns"); } result.push_back(row); } - return move(result); + return result; } TimeSeriesSet TimeSeriesSetImporter::importFromFile(const std::filesystem::path& path, @@ -97,7 +124,7 @@ TimeSeriesSet TimeSeriesSetImporter::importFromFile(const std::filesystem::path& int nSets = csvMatrix[0].size(); for (int i = 0; i < nSets; ++i) { - vector set; + std::vector set; set.reserve(nTimestamps); for (int j = 0; j < nTimestamps; ++j) { diff --git a/src/io/inputs/data-series-csv-importer/include/antares/io/inputs/data-series-csv-importer/DataSeriesRepoImporter.h b/src/io/inputs/data-series-csv-importer/include/antares/io/inputs/data-series-csv-importer/DataSeriesRepoImporter.h index 64e3a49c34..4b7e3b75dc 100644 --- a/src/io/inputs/data-series-csv-importer/include/antares/io/inputs/data-series-csv-importer/DataSeriesRepoImporter.h +++ b/src/io/inputs/data-series-csv-importer/include/antares/io/inputs/data-series-csv-importer/DataSeriesRepoImporter.h @@ -38,11 +38,11 @@ class TimeSeriesSetImporter TimeSeriesSetImporter() = delete; // must not be used static Optimisation::LinearProblemDataImpl::TimeSeriesSet importFromFile( const std::filesystem::path& path, - char csvSeparator = ';'); + char csvSeparator = '\t'); private: static std::vector> csvToMatrix(const std::filesystem::path& path, - char csvSeparator = ';'); + char csvSeparator = '\t'); }; class DataSeriesRepoImporter @@ -51,7 +51,7 @@ class DataSeriesRepoImporter DataSeriesRepoImporter() = delete; // must not be used static Optimisation::LinearProblemDataImpl::DataSeriesRepository importFromDirectory( const std::filesystem::path&, - char csvSeparator = ';'); + char csvSeparator = '\t'); }; } // namespace Antares::IO::Inputs::DataSeriesCsvImporter diff --git a/src/solver/modeler/main.cpp b/src/solver/modeler/main.cpp index 4e04d0b80b..b6625d1f39 100644 --- a/src/solver/modeler/main.cpp +++ b/src/solver/modeler/main.cpp @@ -110,7 +110,7 @@ int main(int argc, const char** argv) try { dataSeriesRepository = IO::Inputs::DataSeriesCsvImporter::DataSeriesRepoImporter:: - importFromDirectory(studyPath / "input" / "data-series", ';'); + importFromDirectory(studyPath / "input" / "data-series", '\t'); logs.info() << "Data-series loaded"; } catch (const std::exception& e) diff --git a/src/tests/resources/Antares_Simulator_Tests_NR b/src/tests/resources/Antares_Simulator_Tests_NR index 0a08ac83d8..7b0dde1c70 160000 --- a/src/tests/resources/Antares_Simulator_Tests_NR +++ b/src/tests/resources/Antares_Simulator_Tests_NR @@ -1 +1 @@ -Subproject commit 0a08ac83d8a687911862272924a21a065efae866 +Subproject commit 7b0dde1c705fa2227bdd1c6fc03e295f42ea4be7 diff --git a/src/tests/src/io/data-series-csv-importer/testDataSeriesCsvImporter.cpp b/src/tests/src/io/data-series-csv-importer/testDataSeriesCsvImporter.cpp index edde4eccc9..17b986d53a 100644 --- a/src/tests/src/io/data-series-csv-importer/testDataSeriesCsvImporter.cpp +++ b/src/tests/src/io/data-series-csv-importer/testDataSeriesCsvImporter.cpp @@ -48,7 +48,7 @@ struct CsvCreationFixture filesystem::path CsvCreationFixture::writeFile(const string filename, const string content) { - auto filepath = temp_path / (filename + ".csv"); + auto filepath = temp_path / (filename); std::ofstream outfile(filepath); outfile << content; outfile.close(); @@ -57,16 +57,48 @@ filesystem::path CsvCreationFixture::writeFile(const string filename, const stri BOOST_FIXTURE_TEST_SUITE(_DataSeriesImport_OneCsvFile_, CsvCreationFixture) -BOOST_AUTO_TEST_CASE(invalid_file) +BOOST_AUTO_TEST_CASE(files_does_not_exist) { - BOOST_CHECK_EXCEPTION(TimeSeriesSetImporter::importFromFile(temp_path / "dummy_123.csv"), + BOOST_CHECK_EXCEPTION(TimeSeriesSetImporter::importFromFile(temp_path / "dummy_123.tsv"), std::invalid_argument, - checkMessage("Could not open file dummy_123.csv")); + checkMessage("Could not open file dummy_123.tsv")); +} + +BOOST_AUTO_TEST_CASE(inconsistent_columns) +{ + auto path = writeFile("wrong.csv", "1;2\n3"); + BOOST_CHECK_EXCEPTION(TimeSeriesSetImporter::importFromFile(path, ';'), + std::invalid_argument, + checkMessage("wrong.csv: rows have inconsistent number of columns")); +} + +BOOST_AUTO_TEST_CASE(inconsistent_rows) +{ + auto path = writeFile("wrong2.csv", "1;2\n;3"); + BOOST_CHECK_EXCEPTION(TimeSeriesSetImporter::importFromFile(path, ';'), + std::invalid_argument, + checkMessage("wrong2.csv: columns have inconsistent number of rows")); +} + +BOOST_AUTO_TEST_CASE(not_a_number) +{ + auto path = writeFile("wrong.csv", "1;2\nXy;3"); + BOOST_CHECK_EXCEPTION(TimeSeriesSetImporter::importFromFile(path, ';'), + std::invalid_argument, + checkMessage("wrong.csv: \"Xy\" is not a number")); +} + +BOOST_AUTO_TEST_CASE(empty_line) +{ + auto path = writeFile("wrong.csv", "1;2\n\n3;4"); + BOOST_CHECK_EXCEPTION(TimeSeriesSetImporter::importFromFile(path, ';'), + std::invalid_argument, + checkMessage("wrong.csv: empty line in the middle of the file")); } BOOST_AUTO_TEST_CASE(empty_file) { - auto path = writeFile("empty", ""); + auto path = writeFile("empty.csv", ""); auto timeSeriesSet = TimeSeriesSetImporter::importFromFile(path); BOOST_CHECK_EQUAL(timeSeriesSet.name(), "empty"); BOOST_CHECK_EXCEPTION(timeSeriesSet.getData(0, 0), @@ -77,7 +109,7 @@ BOOST_AUTO_TEST_CASE(empty_file) BOOST_AUTO_TEST_CASE(one_line_one_column) { - auto path = writeFile("one", "138.583"); + auto path = writeFile("one.tsv", "138.583"); auto timeSeriesSet = TimeSeriesSetImporter::importFromFile(path); BOOST_CHECK_EQUAL(timeSeriesSet.name(), "one"); BOOST_CHECK_EQUAL(timeSeriesSet.getData(0, 0), 138.583); @@ -91,7 +123,7 @@ BOOST_AUTO_TEST_CASE(one_line_one_column) BOOST_AUTO_TEST_CASE(one_line_two_columns) { - auto path = writeFile("one_by_two", "123,456.789"); + auto path = writeFile("one_by_two.csv", "123,456.789\n"); auto timeSeriesSet = TimeSeriesSetImporter::importFromFile(path, ','); BOOST_CHECK_EQUAL(timeSeriesSet.name(), "one_by_two"); BOOST_CHECK_EQUAL(timeSeriesSet.getData(0, 0), 123); @@ -106,7 +138,7 @@ BOOST_AUTO_TEST_CASE(one_line_two_columns) BOOST_AUTO_TEST_CASE(two_lines_one_column) { - auto path = writeFile("two_by_one", "123\n20"); + auto path = writeFile("two_by_one.tsv", "123\n20"); auto timeSeriesSet = TimeSeriesSetImporter::importFromFile(path); BOOST_CHECK_EQUAL(timeSeriesSet.name(), "two_by_one"); BOOST_CHECK_EQUAL(timeSeriesSet.getData(0, 0), 123); @@ -121,8 +153,8 @@ BOOST_AUTO_TEST_CASE(two_lines_one_column) BOOST_AUTO_TEST_CASE(two_lines_two_columns) { - auto path = writeFile("two_by_two", "1;2\n3;4"); - auto timeSeriesSet = TimeSeriesSetImporter::importFromFile(path, ';'); + auto path = writeFile("two_by_two.csv", "1\t2\n3\t4"); + auto timeSeriesSet = TimeSeriesSetImporter::importFromFile(path, '\t'); BOOST_CHECK_EQUAL(timeSeriesSet.name(), "two_by_two"); BOOST_CHECK_EQUAL(timeSeriesSet.getData(0, 0), 1); BOOST_CHECK_EQUAL(timeSeriesSet.getData(0, 1), 3); @@ -151,7 +183,7 @@ BOOST_AUTO_TEST_CASE(empty_dir) BOOST_AUTO_TEST_CASE(one_simple_file) { - writeFile("one", "123"); + writeFile("one.tsv", "123"); auto repo = DataSeriesRepoImporter::importFromDirectory(temp_path); BOOST_CHECK_EQUAL(repo.getDataSeries("one").name(), "one"); BOOST_CHECK_EQUAL(repo.getDataSeries("one").getData(0, 0), 123); @@ -159,8 +191,8 @@ BOOST_AUTO_TEST_CASE(one_simple_file) BOOST_AUTO_TEST_CASE(two_simple_files) { - writeFile("one", "123"); - writeFile("two", "456"); + writeFile("one.csv", "123"); + writeFile("two.tsv", "456"); auto repo = DataSeriesRepoImporter::importFromDirectory(temp_path); BOOST_CHECK_EQUAL(repo.getDataSeries("one").name(), "one"); BOOST_CHECK_EQUAL(repo.getDataSeries("two").name(), "two"); @@ -170,9 +202,9 @@ BOOST_AUTO_TEST_CASE(two_simple_files) BOOST_AUTO_TEST_CASE(three_small_files) { - writeFile("gen1_p_max", "1;2;3;4\n5;6;7;8\n9;10;11;12\n13;14;15;16.17"); - writeFile("gen2_p_max", "10\n20\n30\n40"); - writeFile("node1_load", "20;30\n21;31\n22;32\n23;33"); + writeFile("gen1_p_max.csv", "1\t2\t3\t4\n5\t6\t7\t8\n9\t10\t11\t12\n13\t14\t15\t16.17"); + writeFile("gen2_p_max.tsv", "10\n20\n30\n40"); + writeFile("node1_load.tsv", "20\t30\n21\t31\n22\t32\n23\t33"); auto repo = DataSeriesRepoImporter::importFromDirectory(temp_path); BOOST_CHECK_EQUAL(repo.getDataSeries("gen1_p_max").name(), "gen1_p_max"); diff --git a/src/tests/src/optimisation/linear-problem-data/testLinearProblemData.cpp b/src/tests/src/optimisation/linear-problem-data/testLinearProblemData.cpp index a72f566bec..72d1e0a5da 100644 --- a/src/tests/src/optimisation/linear-problem-data/testLinearProblemData.cpp +++ b/src/tests/src/optimisation/linear-problem-data/testLinearProblemData.cpp @@ -9,14 +9,15 @@ using namespace Antares::Optimisation::LinearProblemDataImpl; -BOOST_AUTO_TEST_CASE(PbData_is_empty__asking_it_a_value_leads_to_exception) +// TODO: activate this test when we support scenario groups +/*BOOST_AUTO_TEST_CASE(PbData_is_empty__asking_it_a_value_leads_to_exception) { LinearProblemData linearProblemData; std::string expected_err_msg = "Scenario group 'group name' does not exist in group repo."; BOOST_CHECK_EXCEPTION(linearProblemData.getData("data set name", "group name", 0, 0), std::invalid_argument, checkMessage(expected_err_msg)); -} +}*/ BOOST_AUTO_TEST_CASE(ask_to_a_simple_linearProblemData_data_it_contains___answer_ok) {