Step/variable-based layout: Frontend implementation

Author: franzpoeschel

include/openPMD/Iteration.hpp

@@ -155,6 +155,7 @@ class Iteration : public Attributable
 
     void flushFileBased(std::string const&, uint64_t);
     void flushGroupBased(uint64_t);
+    void flushStepBased(uint64_t);
     void flush();
     void read();
 

include/openPMD/IterationEncoding.hpp

@@ -31,7 +31,7 @@ namespace openPMD
  */
 enum class IterationEncoding
 {
-    fileBased, groupBased
+    fileBased, groupBased, stepBased
 };
 
 std::ostream&

include/openPMD/backend/Attributable.hpp

@@ -153,7 +153,7 @@ class Attributable
 
 OPENPMD_protected:
     void flushAttributes();
-    void readAttributes();
+    void readAttributes( bool reread = false );
 
     /** Retrieve the value of a floating point Attribute of user-defined precision with ensured type-safety.
      *

src/IO/ADIOS/ADIOS2IOHandler.cpp

@@ -1838,11 +1838,6 @@ namespace detail
             const adios2::Dims & count,
             const bool constantDims )
     {
-        /*
-         * Step/Variable-based iteration layout:
-         * The variable may already be defined from a previous step,
-         * so check if it's already here.
-         */
         adios2::Variable< T > var = IO.InquireVariable< T >( name );
         if( !var )
         {

src/Iteration.cpp

@@ -287,6 +287,21 @@ Iteration::flushGroupBased(uint64_t i)
     flush();
 }
 
+void
+Iteration::flushStepBased( uint64_t i )
+{
+    if( !written() )
+    {
+        /* create iteration path */
+        Parameter< Operation::OPEN_PATH > pOpen;
+        pOpen.path = "";
+        IOHandler->enqueue( IOTask( this, pOpen ) );
+        this->setAttribute( "__step__", i );
+    }
+
+    flush();
+}
+
 void
 Iteration::flush()
 {
@@ -306,8 +321,10 @@ Iteration::flush()
         if( !meshes.empty() || s->containsAttribute("meshesPath") )
         {
             if( !s->containsAttribute("meshesPath") )
+            {
                 s->setMeshesPath("meshes/");
-            s->flushMeshesPath();
+                s->flushMeshesPath();
+            }
             meshes.flush(s->meshesPath());
             for( auto& m : meshes )
                 m.second.flush(m.first);
@@ -316,8 +333,10 @@ Iteration::flush()
         if( !particles.empty() || s->containsAttribute("particlesPath") )
         {
             if( !s->containsAttribute("particlesPath") )
+            {
                 s->setParticlesPath("particles/");
-            s->flushParticlesPath();
+                s->flushParticlesPath();
+            }
             particles.flush(s->particlesPath());
             for( auto& species : particles )
                 species.second.flush(species.first);
@@ -492,6 +511,7 @@ Iteration::beginStep()
             file = this;
             break;
         case IE::groupBased:
+        case IE::stepBased:
             file = &series;
             break;
     }
@@ -503,7 +523,8 @@ Iteration::beginStep()
     }
 
     // re-read -> new datasets might be available
-    if( *series.m_iterationEncoding == IE::groupBased &&
+    if( ( *series.m_iterationEncoding == IE::groupBased ||
+          *series.m_iterationEncoding == IE::stepBased ) &&
         ( this->IOHandler->m_frontendAccess == Access::READ_ONLY ||
           this->IOHandler->m_frontendAccess == Access::READ_WRITE ) )
     {
@@ -536,6 +557,7 @@ Iteration::endStep()
             file = this;
             break;
         case IE::groupBased:
+        case IE::stepBased:
             file = &series;
             break;
     }
@@ -555,6 +577,7 @@ Iteration::getStepStatus()
         case IE::fileBased:
             return *this->m_stepStatus;
         case IE::groupBased:
+        case IE::stepBased:
             return *s->m_stepStatus;
         default:
             throw std::runtime_error( "[Iteration] unreachable" );
@@ -573,6 +596,7 @@ Iteration::setStepStatus( StepStatus status )
             *this->m_stepStatus = status;
             break;
         case IE::groupBased:
+        case IE::stepBased:
             *s->m_stepStatus = status;
             break;
         default:

src/IterationEncoding.cpp

@@ -34,6 +34,9 @@ openPMD::operator<<(std::ostream& os, openPMD::IterationEncoding const& ie)
         case openPMD::IterationEncoding::groupBased:
             os << "groupBased";
             break;
+        case openPMD::IterationEncoding::stepBased:
+            os << "stepBased";
+            break;
     }
     return os;
 }

src/Series.cpp

@@ -305,6 +305,10 @@ Series::setIterationEncoding(IterationEncoding ie)
             setIterationFormat(BASEPATH);
             setAttribute("iterationEncoding", std::string("groupBased"));
             break;
+        case IterationEncoding::stepBased:
+            setIterationFormat(BASEPATH);
+            setAttribute("iterationEncoding", std::string("stepBased"));
+            break;
     }
     return *this;
 }
@@ -321,9 +325,10 @@ Series::setIterationFormat(std::string const& i)
     if( written() )
         throw std::runtime_error("A files iterationFormat can not (yet) be changed after it has been written.");
 
-    if( *m_iterationEncoding == IterationEncoding::groupBased )
+    if( *m_iterationEncoding == IterationEncoding::groupBased ||
+        *m_iterationEncoding == IterationEncoding::stepBased )
         if( basePath() != i && (openPMD() == "1.0.1" || openPMD() == "1.0.0") )
-            throw std::invalid_argument("iterationFormat must not differ from basePath " + basePath() + " for groupBased data");
+            throw std::invalid_argument("iterationFormat must not differ from basePath " + basePath() + " for group- or stepBased data");
 
     setAttribute("iterationFormat", i);
     return *this;
@@ -515,6 +520,7 @@ Series::flush_impl( iterations_iterator begin, iterations_iterator end )
             flushFileBased( begin, end );
             break;
         case IE::groupBased:
+        case IE::stepBased:
             flushGroupBased( begin, end );
             break;
     }
@@ -654,6 +660,7 @@ Series::flushFileBased( iterations_iterator begin, iterations_iterator end )
     }
 }
 
+// @todo rename as flushGroupStepBased
 void
 Series::flushGroupBased( iterations_iterator begin, iterations_iterator end )
 {
@@ -722,7 +729,19 @@ Series::flushGroupBased( iterations_iterator begin, iterations_iterator end )
                 it->second.m_writable->parent = getWritable( &iterations );
                 it->second.parent = getWritable( &iterations );
             }
-            it->second.flushGroupBased(it->first);
+            switch( *m_iterationEncoding )
+            {
+                using IE = IterationEncoding;
+                case IE::groupBased:
+                    it->second.flushGroupBased( it->first );
+                    break;
+                case IE::stepBased:
+                    it->second.flushStepBased( it->first );
+                    break;
+                default:
+                    throw std::runtime_error(
+                        "[Series] Internal control flow error" );
+            }
             if( *it->second.m_closed == Iteration::CloseStatus::ClosedInFrontend )
             {
                 // the iteration has no dedicated file in group-based mode
@@ -805,6 +824,15 @@ Series::readFileBased( )
                                 << "time series with fileBased iteration "
                                     "encoding. Loaded file is groupBased.\n";
                 }
+                else if( encoding == "stepBased" )
+                {
+                    // @todo should we throw? test this path
+                    *m_iterationEncoding = IterationEncoding::stepBased;
+                    std::cerr << "Series constructor called with iteration "
+                                    "regex '%T' suggests loading a "
+                                << "time series with fileBased iteration "
+                                    "encoding. Loaded file is stepBased.\n";
+                }
                 else
                     throw std::runtime_error(
                         "Unknown iterationEncoding: " + encoding );
@@ -859,6 +887,7 @@ Series::readFileBased( )
                                  "Please specify '%0<N>T' or open as read-only.");
 }
 
+// @todo rename as readGroupStepBased
 void
 Series::readGroupBased( bool do_init )
 {
@@ -881,6 +910,8 @@ Series::readGroupBased( bool do_init )
             std::string encoding = Attribute(*aRead.resource).get< std::string >();
             if( encoding == "groupBased" )
                 *m_iterationEncoding = IterationEncoding::groupBased;
+            else if( encoding == "stepBased" )
+                *m_iterationEncoding = IterationEncoding::stepBased;
             else if( encoding == "fileBased" )
             {
                 *m_iterationEncoding = IterationEncoding::fileBased;
@@ -996,23 +1027,50 @@ Series::read()
     IOHandler->enqueue(IOTask(&iterations, pOpen));
 
     readAttributes();
-    iterations.readAttributes();
+    /*
+     * __step__ changes over steps, so reread that.
+     */
+    iterations.readAttributes( /* reread = */ true );
 
     /* obtain all paths inside the basepath (i.e. all iterations) */
-    Parameter< Operation::LIST_PATHS > pList;
-    IOHandler->enqueue(IOTask(&iterations, pList));
-    IOHandler->flush();
-
-    for( auto const& it : *pList.paths )
+    switch(*m_iterationEncoding)
     {
-        Iteration& i = iterations[std::stoull(it)];
-        if ( i.closedByWriter( ) )
+        using IE = IterationEncoding;
+        case IE::groupBased:
+        case IE::fileBased:
         {
-            continue;
+            Parameter< Operation::LIST_PATHS > pList;
+            IOHandler->enqueue(IOTask(&iterations, pList));
+            IOHandler->flush();
+
+            for( auto const& it : *pList.paths )
+            {
+                Iteration& i = iterations[std::stoull(it)];
+                if ( i.closedByWriter( ) )
+                {
+                    continue;
+                }
+                pOpen.path = it;
+                IOHandler->enqueue(IOTask(&i, pOpen));
+                i.read();
+            }
         }
-        pOpen.path = it;
-        IOHandler->enqueue(IOTask(&i, pOpen));
-        i.read();
+        break;
+        case IE::stepBased:
+        {
+            Parameter< Operation::READ_ATT > aRead;
+            uint64_t index = 0;
+            if( iterations.containsAttribute( "__step__" ) )
+            {
+                index = iterations.getAttribute( "__step__" )
+                            .get< uint64_t >();
+            }
+            Iteration & i = iterations[ index ];
+            pOpen.path = "";
+            IOHandler->enqueue( IOTask( &i, pOpen ) );
+            i.read();
+        }
+        break;
     }
 }
 
@@ -1063,6 +1121,7 @@ Series::advance(
     {
         using IE = IterationEncoding;
         case IE::groupBased:
+        case IE::stepBased:
             flushGroupBased( begin, end );
             break;
         case IE::fileBased:
@@ -1131,8 +1190,10 @@ Series::advance(
                 // not closed on a per-iteration basis
                 // We will treat it as such nonetheless
                 *iteration.m_closed = Iteration::CloseStatus::ClosedInBackend;
+                break;
             }
-            break;
+            case IE::stepBased: // no action necessary
+                break;
         }
     }
 
@@ -1157,7 +1218,9 @@ Series::openIteration( uint64_t index, Iteration iteration )
     pOpen.path = auxiliary::replace_first( basePath(), "%T/", "" );
     IOHandler->enqueue( IOTask( &iterations, pOpen ) );
     /* open iteration path */
-    pOpen.path = std::to_string( index );
+    pOpen.path = *m_iterationEncoding == IterationEncoding::stepBased
+        ? ""
+        : std::to_string( index );
     IOHandler->enqueue( IOTask( &iteration, pOpen ) );
     switch( *iteration.m_closed )
     {
@@ -1297,6 +1360,7 @@ SeriesIterator::operator++()
     {
         using IE = IterationEncoding;
         case IE::groupBased:
+        case IE::stepBased:
         {
             // since we are in group-based iteration layout, it does not
             // matter which iteration we begin a step upon

src/backend/Attributable.cpp

@@ -142,21 +142,31 @@ Attributable::flushAttributes()
 }
 
 void
-Attributable::readAttributes()
+Attributable::readAttributes( bool reread )
 {
     Parameter< Operation::LIST_ATTS > aList;
     IOHandler->enqueue(IOTask(this, aList));
     IOHandler->flush();
     std::vector< std::string > written_attributes = attributes();
 
-    /* std::set_difference requires sorted ranges */
-    std::sort(aList.attributes->begin(), aList.attributes->end());
-    std::sort(written_attributes.begin(), written_attributes.end());
-
     std::set< std::string > tmpAttributes;
-    std::set_difference(aList.attributes->begin(), aList.attributes->end(),
-                        written_attributes.begin(), written_attributes.end(),
-                        std::inserter(tmpAttributes, tmpAttributes.begin()));
+    if( reread )
+    {
+        tmpAttributes = std::set< std::string >(
+            aList.attributes->begin(),
+            aList.attributes->end() );
+    }
+    else
+    {
+        /* std::set_difference requires sorted ranges */
+        std::sort(aList.attributes->begin(), aList.attributes->end());
+        std::sort(written_attributes.begin(), written_attributes.end());
+
+        std::set_difference(
+            aList.attributes->begin(), aList.attributes->end(),
+            written_attributes.begin(), written_attributes.end(),
+            std::inserter(tmpAttributes, tmpAttributes.begin()));
+    }
 
     using DT = Datatype;
     Parameter< Operation::READ_ATT > aRead;

src/binding/python/IterationEncoding.cpp

@@ -31,5 +31,6 @@ void init_IterationEncoding(py::module &m) {
     py::enum_<IterationEncoding>(m, "Iteration_Encoding")
         .value("file_based", IterationEncoding::fileBased)
         .value("group_based", IterationEncoding::groupBased)
+        .value("step_based", IterationEncoding::stepBased)
     ;
 }