Automatically use new attribute layout if using step-based encoding

Also automatically use steps if using new attribute layout

ADIOS2 schema

Automatically detect layout when reading

Make automatic recognition of access mode more intelligent

Previous commits uncovered an old bug

ADIOS2: Use YYYYMMDD schema tag

Also fix a bug in Parallel IO tests that I noticed

Undo unnecessary whitespace changes

Author: franzpoeschel

include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp

@@ -32,6 +32,7 @@
 #include "openPMD/auxiliary/Option.hpp"
 #include "openPMD/backend/Writable.hpp"
 #include "openPMD/config.hpp"
+#include "openPMD/IterationEncoding.hpp"
 
 #if openPMD_HAVE_ADIOS2
 #    include <adios2.h>
@@ -213,6 +214,7 @@ class ADIOS2IOHandlerImpl
 
 private:
     adios2::ADIOS m_ADIOS;
+    IterationEncoding m_iterationEncoding = IterationEncoding::groupBased;
     /**
      * The ADIOS2 engine type, to be passed to adios2::IO::SetEngine
      */
@@ -1018,6 +1020,7 @@ namespace detail
         detail::AttributeReader const m_attributeReader;
         PreloadAdiosAttributes preloadAttributes;
         ADIOS2Schema::schema_t & m_schema;
+        IterationEncoding & m_iterationEncoding;
 
         /*
          * We call an attribute committed if the step during which it was
@@ -1270,7 +1273,7 @@ class ADIOS2IOHandler : public AbstractIOHandler
 {
 #if openPMD_HAVE_ADIOS2
 
-friend class ADIOS2IOHandlerImpl;
+    friend class ADIOS2IOHandlerImpl;
 
 private:
     ADIOS2IOHandlerImpl m_impl;
@@ -1289,7 +1292,7 @@ friend class ADIOS2IOHandlerImpl;
         }
         catch( ... )
         {
-            std::cerr << "[~ADIOS2IOHandler] An error occurred." << std::endl;
+             std::cerr << "[~ADIOS2IOHandler] An error occurred." << std::endl;
         }
     }
 

include/openPMD/IO/IOTask.hpp

@@ -25,6 +25,7 @@
 #include "openPMD/backend/Attribute.hpp"
 #include "openPMD/ChunkInfo.hpp"
 #include "openPMD/Dataset.hpp"
+#include "openPMD/IterationEncoding.hpp"
 
 #include <memory>
 #include <map>
@@ -107,7 +108,8 @@ template<>
 struct OPENPMDAPI_EXPORT Parameter< Operation::CREATE_FILE > : public AbstractParameter
 {
     Parameter() = default;
-    Parameter(Parameter const & p) : AbstractParameter(), name(p.name) {}
+    Parameter(Parameter const & p) :
+        AbstractParameter(), name(p.name), encoding(p.encoding) {}
 
     std::unique_ptr< AbstractParameter >
     clone() const override
@@ -117,13 +119,15 @@ struct OPENPMDAPI_EXPORT Parameter< Operation::CREATE_FILE > : public AbstractPa
     }
 
     std::string name = "";
+    IterationEncoding encoding = IterationEncoding::groupBased;
 };
 
 template<>
 struct OPENPMDAPI_EXPORT Parameter< Operation::OPEN_FILE > : public AbstractParameter
 {
     Parameter() = default;
-    Parameter(Parameter const & p) : AbstractParameter(), name(p.name) {}
+    Parameter(Parameter const & p) :
+        AbstractParameter(), name(p.name), encoding(p.encoding) {}
 
     std::unique_ptr< AbstractParameter >
     clone() const override
@@ -133,6 +137,7 @@ struct OPENPMDAPI_EXPORT Parameter< Operation::OPEN_FILE > : public AbstractPara
     }
 
     std::string name = "";
+    IterationEncoding encoding = IterationEncoding::groupBased;
 };
 
 template<>

src/IO/ADIOS/ADIOS2IOHandler.cpp

@@ -289,6 +289,7 @@ void ADIOS2IOHandlerImpl::createFile(
             VERIFY( success, "[ADIOS2] Could not create directory." );
         }
 
+        m_iterationEncoding = parameters.encoding;
         associateWithFile( writable, shared_name );
         this->m_dirty.emplace( shared_name );
         getFileData( shared_name ).m_mode = adios2::Mode::Write; // WORKAROUND
@@ -427,6 +428,7 @@ void ADIOS2IOHandlerImpl::openFile(
     writable->written = true;
     writable->abstractFilePosition = std::make_shared< ADIOS2FilePosition >( );
 
+    m_iterationEncoding = parameters.encoding;
     // enforce opening the file
     // lazy opening is deathly in parallel situations
     getFileData( file );
@@ -1931,6 +1933,7 @@ namespace detail
         , m_readDataset( &impl )
         , m_attributeReader()
         , m_schema( impl.m_schema )
+        , m_iterationEncoding( impl.m_iterationEncoding )
         , m_engineType( impl.m_engineType )
     {
         if( !m_IO )
@@ -2002,6 +2005,12 @@ namespace detail
             "bp4", "bp3", "hdf5", "file"
         };
 
+        // step/variable-based iteration encoding requires the new schema
+        if( m_iterationEncoding == IterationEncoding::stepBased )
+        {
+            m_schema = ADIOS2Schema::schema_2021_02_09;
+        }
+
         // set engine type
         bool isStreaming = false;
         {

src/Iteration.cpp

@@ -224,6 +224,7 @@ Iteration::flushFileBased(std::string const& filename, uint64_t i)
         {
             Parameter< Operation::OPEN_FILE > fOpen;
             fOpen.name = filename;
+            fOpen.encoding = IterationEncoding::fileBased;
             IOHandler()->enqueue(IOTask(s, fOpen));
             flush();
 

src/Series.cpp

@@ -649,6 +649,7 @@ SeriesImpl::flushGroupBased( iterations_iterator begin, iterations_iterator end
         {
             Parameter< Operation::CREATE_FILE > fCreate;
             fCreate.name = *series.m_name;
+            fCreate.encoding = iterationEncoding();
             IOHandler()->enqueue(IOTask(this, fCreate));
         }
 
@@ -735,6 +736,7 @@ SeriesImpl::readFileBased( )
     Parameter< Operation::OPEN_FILE > fOpen;
     Parameter< Operation::CLOSE_FILE > fClose;
     Parameter< Operation::READ_ATT > aRead;
+    fOpen.encoding = iterationEncoding();
 
     if( !auxiliary::directory_exists(IOHandler()->directory) )
         throw no_such_file_error("Supplied directory is not valid: " + IOHandler()->directory);
@@ -847,6 +849,7 @@ SeriesImpl::readGroupBased( bool do_init )
 {
     auto & series = get();
     Parameter< Operation::OPEN_FILE > fOpen;
+    fOpen.encoding = iterationEncoding();
     fOpen.name = *series.m_name;
     IOHandler()->enqueue(IOTask(this, fOpen));
     IOHandler()->flush();
@@ -1174,6 +1177,7 @@ SeriesImpl::openIteration( uint64_t index, Iteration iteration )
     auto & series = get();
     // open the iteration's file again
     Parameter< Operation::OPEN_FILE > fOpen;
+    fOpen.encoding = iterationEncoding();
     fOpen.name = iterationFilename( index );
     IOHandler()->enqueue( IOTask( this, fOpen ) );
 

test/SerialIOTest.cpp

@@ -3296,20 +3296,9 @@ TEST_CASE( "serial_iterator", "[serial][adios2]" )
 void
 stepBasedSingleIteration( std::string const & file )
 {
-    std::string config = R"END(
-{
-    "adios2":
-    {
-        "new_attribute_layout": true,
-        "engine":
-        {
-            "usesteps": true
-        }
-    }
-})END";
     constexpr Extent::value_type extent = 1000;
     {
-        Series writeSeries( file, Access::CREATE, config );
+        Series writeSeries( file, Access::CREATE );
         writeSeries.setIterationEncoding( IterationEncoding::stepBased );
         auto iterations = writeSeries.writeIterations();
         auto iteration = writeSeries.iterations[ 0 ];
@@ -3323,7 +3312,7 @@ stepBasedSingleIteration( std::string const & file )
     }
 
     {
-        Series readSeries( file, Access::READ_ONLY, config );
+        Series readSeries( file, Access::READ_ONLY );
 
         auto E_x = readSeries.iterations[ 0 ].meshes[ "E" ][ "x" ];
         REQUIRE( E_x.getDimensionality() == 1 );
@@ -3332,7 +3321,7 @@ stepBasedSingleIteration( std::string const & file )
         readSeries.flush();
         for( size_t i = 0; i < extent; ++i )
         {
-            REQUIRE( chunk.get()[ i ] == i );
+            REQUIRE( chunk.get()[ i ] == int( i ) );
         }
     }
 }
@@ -3349,20 +3338,9 @@ TEST_CASE( "stepBasedSingleIteration", "[serial][adios2]" )
 void
 stepBasedSeries( std::string const & file )
 {
-    std::string config = R"END(
-{
-    "adios2":
-    {
-        "new_attribute_layout": true,
-        "engine":
-        {
-            "usesteps": true
-        }
-    }
-})END";
     constexpr Extent::value_type extent = 1000;
     {
-        Series writeSeries( file, Access::CREATE, config );
+        Series writeSeries( file, Access::CREATE );
         if( writeSeries.backend() == "ADIOS1" )
         {
             return;
@@ -3381,7 +3359,7 @@ stepBasedSeries( std::string const & file )
         }
     }
 
-    Series readSeries( file, Access::READ_ONLY, config );
+    Series readSeries( file, Access::READ_ONLY );
 
     size_t last_iteration_index = 0;
     for( auto iteration : readSeries.readIterations() )
@@ -3393,7 +3371,7 @@ stepBasedSeries( std::string const & file )
         iteration.close();
         for( size_t i = 0; i < extent; ++i )
         {
-            REQUIRE( chunk.get()[ i ] == iteration.iterationIndex );
+            REQUIRE( chunk.get()[ i ] == int( iteration.iterationIndex ) );
         }
         last_iteration_index = iteration.iterationIndex;
     }
@@ -3406,6 +3384,7 @@ TEST_CASE( "stepBasedSeries", "[serial][adios2]" )
 }
 #endif
 
+// @todo Upon switching to ADIOS2 2.7.0, test this the other way around also
 void
 iterate_nonstreaming_series( std::string const & file )
 {