Step/variable-based layout: Frontend implementation

Author: franzpoeschel

include/openPMD/Iteration.hpp

@@ -155,6 +155,7 @@ class Iteration : public LegacyAttributable
 
     void flushFileBased(std::string const&, uint64_t);
     void flushGroupBased(uint64_t);
+    void flushStepBased(uint64_t);
     void flush();
     void read();
 

include/openPMD/IterationEncoding.hpp

@@ -31,7 +31,7 @@ namespace openPMD
  */
 enum class IterationEncoding
 {
-    fileBased, groupBased
+    fileBased, groupBased, stepBased
 };
 
 std::ostream&

include/openPMD/backend/Attributable.hpp

@@ -204,7 +204,7 @@ class AttributableImpl
     internal::SeriesInternal & retrieveSeries();
 
     void flushAttributes();
-    void readAttributes();
+    void readAttributes( bool reread = false );
 
     /** Retrieve the value of a floating point Attribute of user-defined precision with ensured type-safety.
      *

src/Iteration.cpp

@@ -252,6 +252,21 @@ Iteration::flushGroupBased(uint64_t i)
     flush();
 }
 
+void
+Iteration::flushStepBased( uint64_t i )
+{
+    if( !written() )
+    {
+        /* create iteration path */
+        Parameter< Operation::OPEN_PATH > pOpen;
+        pOpen.path = "";
+        IOHandler()->enqueue( IOTask( this, pOpen ) );
+        this->setAttribute( "__step__", i );
+    }
+
+    flush();
+}
+
 void
 Iteration::flush()
 {
@@ -270,8 +285,10 @@ Iteration::flush()
         if( !meshes.empty() || s->containsAttribute("meshesPath") )
         {
             if( !s->containsAttribute("meshesPath") )
+            {
                 s->setMeshesPath("meshes/");
-            s->flushMeshesPath();
+                s->flushMeshesPath();
+            }
             meshes.flush(s->meshesPath());
             for( auto& m : meshes )
                 m.second.flush(m.first);
@@ -280,8 +297,10 @@ Iteration::flush()
         if( !particles.empty() || s->containsAttribute("particlesPath") )
         {
             if( !s->containsAttribute("particlesPath") )
+            {
                 s->setParticlesPath("particles/");
-            s->flushParticlesPath();
+                s->flushParticlesPath();
+            }
             particles.flush(s->particlesPath());
             for( auto& species : particles )
                 species.second.flush(species.first);
@@ -454,6 +473,7 @@ Iteration::beginStep()
             file = m_attributableData.get();
             break;
         case IE::groupBased:
+        case IE::stepBased:
             file = &series;
             break;
     }
@@ -465,7 +485,8 @@ Iteration::beginStep()
     }
 
     // re-read -> new datasets might be available
-    if( series.iterationEncoding() == IE::groupBased &&
+    if( ( series.iterationEncoding() == IE::groupBased ||
+          series.iterationEncoding() == IE::stepBased ) &&
         ( this->IOHandler()->m_frontendAccess == Access::READ_ONLY ||
           this->IOHandler()->m_frontendAccess == Access::READ_WRITE ) )
     {
@@ -497,6 +518,7 @@ Iteration::endStep()
             file = m_attributableData.get();
             break;
         case IE::groupBased:
+        case IE::stepBased:
             file = &series;
             break;
     }
@@ -515,6 +537,7 @@ Iteration::getStepStatus()
         case IE::fileBased:
             return *this->m_stepStatus;
         case IE::groupBased:
+        case IE::stepBased:
             return *s->m_stepStatus;
         default:
             throw std::runtime_error( "[Iteration] unreachable" );
@@ -532,6 +555,7 @@ Iteration::setStepStatus( StepStatus status )
             *this->m_stepStatus = status;
             break;
         case IE::groupBased:
+        case IE::stepBased:
             *s->m_stepStatus = status;
             break;
         default:

src/IterationEncoding.cpp

@@ -34,6 +34,9 @@ openPMD::operator<<(std::ostream& os, openPMD::IterationEncoding const& ie)
         case openPMD::IterationEncoding::groupBased:
             os << "groupBased";
             break;
+        case openPMD::IterationEncoding::stepBased:
+            os << "stepBased";
+            break;
     }
     return os;
 }

src/Series.cpp

@@ -270,6 +270,10 @@ SeriesImpl::setIterationEncoding(IterationEncoding ie)
             setIterationFormat(BASEPATH);
             setAttribute("iterationEncoding", std::string("groupBased"));
             break;
+        case IterationEncoding::stepBased:
+            setIterationFormat(BASEPATH);
+            setAttribute("iterationEncoding", std::string("stepBased"));
+            break;
     }
     return *this;
 }
@@ -283,13 +287,13 @@ SeriesImpl::iterationFormat() const
 SeriesImpl&
 SeriesImpl::setIterationFormat(std::string const& i)
 {
-    auto & series = get();
     if( written() )
         throw std::runtime_error("A files iterationFormat can not (yet) be changed after it has been written.");
 
-    if( *series.m_iterationEncoding == IterationEncoding::groupBased )
+    if( iterationEncoding() == IterationEncoding::groupBased ||
+        iterationEncoding() == IterationEncoding::stepBased )
         if( basePath() != i && (openPMD() == "1.0.1" || openPMD() == "1.0.0") )
-            throw std::invalid_argument("iterationFormat must not differ from basePath " + basePath() + " for groupBased data");
+            throw std::invalid_argument("iterationFormat must not differ from basePath " + basePath() + " for group- or stepBased data");
 
     setAttribute("iterationFormat", i);
     return *this;
@@ -467,6 +471,7 @@ SeriesImpl::flush_impl( iterations_iterator begin, iterations_iterator end )
             flushFileBased( begin, end );
             break;
         case IE::groupBased:
+        case IE::stepBased:
             flushGroupBased( begin, end );
             break;
     }
@@ -607,6 +612,7 @@ SeriesImpl::flushFileBased( iterations_iterator begin, iterations_iterator end )
     }
 }
 
+// @todo rename as flushGroupStepBased
 void
 SeriesImpl::flushGroupBased( iterations_iterator begin, iterations_iterator end )
 {
@@ -675,7 +681,19 @@ SeriesImpl::flushGroupBased( iterations_iterator begin, iterations_iterator end
             {
                 it->second.parent() = getWritable( &series.iterations );
             }
-            it->second.flushGroupBased(it->first);
+            switch( iterationEncoding() )
+            {
+                using IE = IterationEncoding;
+                case IE::groupBased:
+                    it->second.flushGroupBased( it->first );
+                    break;
+                case IE::stepBased:
+                    it->second.flushStepBased( it->first );
+                    break;
+                default:
+                    throw std::runtime_error(
+                        "[Series] Internal control flow error" );
+            }
             if( *it->second.m_closed == Iteration::CloseStatus::ClosedInFrontend )
             {
                 // the iteration has no dedicated file in group-based mode
@@ -760,6 +778,15 @@ SeriesImpl::readFileBased( )
                                 << "time series with fileBased iteration "
                                     "encoding. Loaded file is groupBased.\n";
                 }
+                else if( encoding == "stepBased" )
+                {
+                    // @todo should we throw? test this path
+                    *series.m_iterationEncoding = IterationEncoding::stepBased;
+                    std::cerr << "Series constructor called with iteration "
+                                    "regex '%T' suggests loading a "
+                                << "time series with fileBased iteration "
+                                    "encoding. Loaded file is stepBased.\n";
+                }
                 else
                     throw std::runtime_error(
                         "Unknown iterationEncoding: " + encoding );
@@ -814,6 +841,7 @@ SeriesImpl::readFileBased( )
                                  "Please specify '%0<N>T' or open as read-only.");
 }
 
+// @todo rename as readGroupStepBased
 void
 SeriesImpl::readGroupBased( bool do_init )
 {
@@ -837,6 +865,8 @@ SeriesImpl::readGroupBased( bool do_init )
             std::string encoding = Attribute(*aRead.resource).get< std::string >();
             if( encoding == "groupBased" )
                 *series.m_iterationEncoding = IterationEncoding::groupBased;
+            else if( encoding == "stepBased" )
+                *series.m_iterationEncoding = IterationEncoding::stepBased;
             else if( encoding == "fileBased" )
             {
                 *series.m_iterationEncoding = IterationEncoding::fileBased;
@@ -954,23 +984,50 @@ SeriesImpl::read()
     IOHandler()->enqueue(IOTask(&series.iterations, pOpen));
 
     readAttributes();
-    series.iterations.readAttributes();
+    /*
+     * __step__ changes over steps, so reread that.
+     */
+    series.iterations.readAttributes( /* reread = */ true );
 
     /* obtain all paths inside the basepath (i.e. all iterations) */
-    Parameter< Operation::LIST_PATHS > pList;
-    IOHandler()->enqueue(IOTask(&series.iterations, pList));
-    IOHandler()->flush();
-
-    for( auto const& it : *pList.paths )
+    switch(*series.m_iterationEncoding)
     {
-        Iteration& i = series.iterations[std::stoull(it)];
-        if ( i.closedByWriter( ) )
+        using IE = IterationEncoding;
+        case IE::groupBased:
+        case IE::fileBased:
         {
-            continue;
+            Parameter< Operation::LIST_PATHS > pList;
+            IOHandler()->enqueue(IOTask(&series.iterations, pList));
+            IOHandler()->flush();
+
+            for( auto const& it : *pList.paths )
+            {
+                Iteration& i = series.iterations[std::stoull(it)];
+                if ( i.closedByWriter( ) )
+                {
+                    continue;
+                }
+                pOpen.path = it;
+                IOHandler()->enqueue(IOTask(&i, pOpen));
+                i.read();
+            }
         }
-        pOpen.path = it;
-        IOHandler()->enqueue(IOTask(&i, pOpen));
-        i.read();
+        break;
+        case IE::stepBased:
+        {
+            Parameter< Operation::READ_ATT > aRead;
+            uint64_t index = 0;
+            if( series.iterations.containsAttribute( "__step__" ) )
+            {
+                index = series.iterations.getAttribute( "__step__" )
+                            .get< uint64_t >();
+            }
+            Iteration & i = series.iterations[ index ];
+            pOpen.path = "";
+            IOHandler()->enqueue( IOTask( &i, pOpen ) );
+            i.read();
+        }
+        break;
     }
 }
 
@@ -1027,6 +1084,7 @@ SeriesImpl::advance(
     {
         using IE = IterationEncoding;
         case IE::groupBased:
+        case IE::stepBased:
             flushGroupBased( begin, end );
             break;
         case IE::fileBased:
@@ -1095,8 +1153,10 @@ SeriesImpl::advance(
                 // not closed on a per-iteration basis
                 // We will treat it as such nonetheless
                 *iteration.m_closed = Iteration::CloseStatus::ClosedInBackend;
+                break;
             }
-            break;
+            case IE::stepBased: // no action necessary
+                break;
         }
     }
 
@@ -1122,7 +1182,9 @@ SeriesImpl::openIteration( uint64_t index, Iteration iteration )
     pOpen.path = auxiliary::replace_first( basePath(), "%T/", "" );
     IOHandler()->enqueue( IOTask( &series.iterations, pOpen ) );
     /* open iteration path */
-    pOpen.path = std::to_string( index );
+    pOpen.path = iterationEncoding() == IterationEncoding::stepBased
+        ? ""
+        : std::to_string( index );
     IOHandler()->enqueue( IOTask( &iteration, pOpen ) );
     switch( *iteration.m_closed )
     {
@@ -1367,6 +1429,7 @@ SeriesIterator::operator++()
     {
         using IE = IterationEncoding;
         case IE::groupBased:
+        case IE::stepBased:
         {
             // since we are in group-based iteration layout, it does not
             // matter which iteration we begin a step upon

src/backend/Attributable.cpp

@@ -153,21 +153,31 @@ AttributableImpl::flushAttributes()
 }
 
 void
-AttributableImpl::readAttributes()
+AttributableImpl::readAttributes( bool reread )
 {
     Parameter< Operation::LIST_ATTS > aList;
     IOHandler()->enqueue(IOTask(this, aList));
     IOHandler()->flush();
     std::vector< std::string > written_attributes = attributes();
 
-    /* std::set_difference requires sorted ranges */
-    std::sort(aList.attributes->begin(), aList.attributes->end());
-    std::sort(written_attributes.begin(), written_attributes.end());
-
     std::set< std::string > tmpAttributes;
-    std::set_difference(aList.attributes->begin(), aList.attributes->end(),
-                        written_attributes.begin(), written_attributes.end(),
-                        std::inserter(tmpAttributes, tmpAttributes.begin()));
+    if( reread )
+    {
+        tmpAttributes = std::set< std::string >(
+            aList.attributes->begin(),
+            aList.attributes->end() );
+    }
+    else
+    {
+        /* std::set_difference requires sorted ranges */
+        std::sort(aList.attributes->begin(), aList.attributes->end());
+        std::sort(written_attributes.begin(), written_attributes.end());
+
+        std::set_difference(
+            aList.attributes->begin(), aList.attributes->end(),
+            written_attributes.begin(), written_attributes.end(),
+            std::inserter(tmpAttributes, tmpAttributes.begin()));
+    }
 
     using DT = Datatype;
     Parameter< Operation::READ_ATT > aRead;

src/binding/python/IterationEncoding.cpp

@@ -31,5 +31,6 @@ void init_IterationEncoding(py::module &m) {
     py::enum_<IterationEncoding>(m, "Iteration_Encoding")
         .value("file_based", IterationEncoding::fileBased)
         .value("group_based", IterationEncoding::groupBased)
+        .value("step_based", IterationEncoding::stepBased)
     ;
 }