Backend additions

1) New streaming status: RANDOM_ACCESS, for non-streaming situations
2) Variable attributes, to be written only if the backend has support
   for steps

Author: franzpoeschel

include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp

@@ -1225,13 +1225,6 @@ namespace detail
         void
         invalidateVariablesMap();
 
-    private:
-        ADIOS2IOHandlerImpl * m_impl;
-        auxiliary::Option< adios2::Engine > m_engine; //! ADIOS engine
-        /**
-         * The ADIOS2 engine type, to be passed to adios2::IO::SetEngine
-         */
-        std::string m_engineType;
         /*
          * streamStatus is NoStream for file-based ADIOS engines.
          * This is relevant for the method BufferedActions::requireActiveStep,
@@ -1322,6 +1315,14 @@ namespace detail
             Undecided
         };
         StreamStatus streamStatus = StreamStatus::OutsideOfStep;
+
+    private:
+        ADIOS2IOHandlerImpl * m_impl;
+        auxiliary::Option< adios2::Engine > m_engine; //! ADIOS engine
+        /**
+         * The ADIOS2 engine type, to be passed to adios2::IO::SetEngine
+         */
+        std::string m_engineType;
         adios2::StepStatus m_lastStepStatus = adios2::StepStatus::OK;
 
         /**

include/openPMD/IO/AbstractIOHandlerImpl.hpp

@@ -156,8 +156,10 @@ class AbstractIOHandlerImpl
    * The return status code shall be stored as parameters.status.
    */
   virtual void
-  advance( Writable *, Parameter< Operation::ADVANCE > & )
-  {}
+  advance( Writable *, Parameter< Operation::ADVANCE > & parameters )
+  {
+    *parameters.status = AdvanceStatus::RANDOMACCESS;
+  }
 
   /** Close an openPMD group.
    *
@@ -336,6 +338,11 @@ class AbstractIOHandlerImpl
    * The attribute should be of datatype parameters.dtype.
    * Any existing attribute with the same name should be overwritten. If possible, only the value should be changed if the datatype stays the same.
    * The attribute should be written to physical storage after the operation completes successfully.
+   * If the parameter changesOverSteps is true, then the attribute must be able
+   * to hold different values across IO steps. If the backend does not support
+   * IO steps in such a way, the attribute should not be written.
+   * (IO steps are an optional backend feature and the frontend must implement
+   * fallback measures in such a case)
    * All datatypes of Datatype should be supported in a type-safe way.
    */
   virtual void writeAttribute(Writable*, Parameter< Operation::WRITE_ATT > const&) = 0;

include/openPMD/IO/IOTask.hpp

@@ -485,7 +485,8 @@ struct OPENPMDAPI_EXPORT Parameter< Operation::WRITE_ATT > : public AbstractPara
 {
     Parameter() = default;
     Parameter(Parameter const & p) : AbstractParameter(),
-        name(p.name), dtype(p.dtype), resource(p.resource) {}
+        name(p.name), dtype(p.dtype), changesOverSteps(p.changesOverSteps),
+        resource(p.resource) {}
 
     std::unique_ptr< AbstractParameter >
     clone() const override
@@ -496,6 +497,7 @@ struct OPENPMDAPI_EXPORT Parameter< Operation::WRITE_ATT > : public AbstractPara
 
     std::string name = "";
     Datatype dtype = Datatype::UNDEFINED;
+    bool changesOverSteps = false;
     Attribute::resource resource;
 };
 

include/openPMD/Streaming.hpp

@@ -19,8 +19,9 @@ namespace openPMD
  */
 enum class AdvanceStatus : unsigned char
 {
-    OK,  /* stream goes on */
-    OVER /* stream is over */
+    OK,   /* stream goes on */
+    OVER, /* stream is over */
+    RANDOMACCESS /* there is no stream, it will never be over */
 };
 
 /**

src/IO/ADIOS/ADIOS2IOHandler.cpp

@@ -617,6 +617,11 @@ void ADIOS2IOHandlerImpl::writeAttribute(
     switch( attributeLayout() )
     {
         case AttributeLayout::ByAdiosAttributes:
+            if( parameters.changesOverSteps )
+            {
+                // cannot do this
+                return;
+            }
             switchType< detail::OldAttributeWriter >(
                 parameters.dtype,
                 this,
@@ -634,6 +639,13 @@ void ADIOS2IOHandlerImpl::writeAttribute(
             auto prefix = filePositionToString( pos );
 
             auto & filedata = getFileData( file, IfFileNotOpen::ThrowError );
+            if( parameters.changesOverSteps &&
+                filedata.streamStatus ==
+                    detail::BufferedActions::StreamStatus::NoStream )
+            {
+                // cannot do this
+                return;
+            }
             filedata.invalidateAttributesMap();
             m_dirty.emplace( std::move( file ) );
 
@@ -2785,7 +2797,7 @@ namespace detail
             m_IO.DefineAttribute< bool_representation >(
                 ADIOS2Defaults::str_usesstepsAttribute, 0 );
             flush( FlushLevel::UserFlush, /* writeAttributes = */ false );
-            return AdvanceStatus::OK;
+            return AdvanceStatus::RANDOMACCESS;
         }
 
         m_IO.DefineAttribute< bool_representation >(

src/IO/ADIOS/CommonADIOS1IOHandler.cpp

@@ -1052,6 +1052,11 @@ void
 CommonADIOS1IOHandlerImpl< ChildClass >::writeAttribute(Writable* writable,
                                     Parameter< Operation::WRITE_ATT > const& parameters)
 {
+    if( parameters.changesOverSteps )
+    {
+        // cannot do this
+        return;
+    }
     if( m_handler->m_backendAccess == Access::READ_ONLY )
         throw std::runtime_error("[ADIOS1] Writing an attribute in a file opened as read only is not possible.");
 

src/IO/HDF5/HDF5IOHandler.cpp

@@ -1044,6 +1044,11 @@ void
 HDF5IOHandlerImpl::writeAttribute(Writable* writable,
                                   Parameter< Operation::WRITE_ATT > const& parameters)
 {
+    if( parameters.changesOverSteps )
+    {
+        // cannot do this
+        return;
+    }
     if( m_handler->m_backendAccess == Access::READ_ONLY )
         throw std::runtime_error("[HDF5] Writing an attribute in a file opened as read only is not possible.");
 

src/IO/JSON/JSONIOHandlerImpl.cpp

@@ -907,6 +907,11 @@ namespace openPMD
         Parameter< Operation::WRITE_ATT > const & parameter
     )
     {
+        if( parameter.changesOverSteps )
+        {
+            // cannot do this
+            return;
+        }
         if(m_handler->m_backendAccess == Access::READ_ONLY )
         {
             throw std::runtime_error( "[JSON] Creating a dataset in a file opened as read only is not possible." );