Backports from master

bindings/C/adios2/c/adios2_c_io.h

@@ -329,8 +329,8 @@ adios2_error adios2_remove_all_attributes(adios2_io *io);
  * MPI Collective function as it calls MPI_Comm_dup
  * @param io engine owner
  * @param name unique engine identifier
- * @param mode adios2_mode_write, adios2_mode_read, adios2_mode_append, and
- * adios2_mode_readRandomAccess
+ * @param mode adios2_mode_write, adios2_mode_read, adios2_mode_append
+ * and adios2_mode_readRandomAccess
  * @return success: handler, failure: NULL
  */
 adios2_engine *adios2_open(adios2_io *io, const char *name, const adios2_mode mode);
@@ -341,7 +341,7 @@ adios2_engine *adios2_open(adios2_io *io, const char *name, const adios2_mode mo
  * MPI Collective function as it calls MPI_Comm_dup
  * @param io engine owner
  * @param name unique engine identifier
- * @param mode adios2_mode_write, adios2_mode_read, adios2_mode_append, and
+ * @param mode adios2_mode_write, adios2_mode_read, adios2_mode_append and
  * adios2_mode_readRandomAccess
  * @param comm communicator other than adios' handler comm. MPI only.
  * @return success: handler, failure: NULL

bindings/CXX11/adios2/cxx11/Variable.h

@@ -214,7 +214,7 @@ class Variable
      * variable.Count() = {Ny,Nx}, then memoryCount = {Ny+2,Nx+2}
      * </pre>
      */
-    void SetMemorySelection(const adios2::Box<adios2::Dims> &memorySelection);
+    void SetMemorySelection(const adios2::Box<adios2::Dims> &memorySelection = {{}, {}});
 
     /**
      * Sets a step selection modifying current startStep, countStep

docs/user_guide/source/components/anatomy.rst

@@ -114,7 +114,10 @@ named `adios2::Mode::ReadRandomAccess`. `adios2::Mode::Read` mode allows data ac
 current timestep. `ReadRandomAccess` can only be used with file engines and involves loading all the file metadata at
 once. So it can be more memory intensive than `adios2::Mode::Read` mode, but allows reading data from any timestep using
 `SetStepSelection()`. If you use `adios2::Mode::ReadRandomAccess` mode, be sure to allocate enough memory to hold
-multiple steps of the variable content.
+multiple steps of the variable content.  Note that ADIOS streaming
+engines (like SST, DataMan, etc.) do not support `ReadRandomAccess`
+mode.  Also newer file Engines like BP5 to not allow
+`BeginStep/EndStep` calls in `ReadRandomAccess` mode.
 
 .. code:: C++
 
@@ -134,3 +137,35 @@ multiple steps of the variable content.
     |   |--> IO goes out of scope
     |
     |--> ADIOS goes out of scope or adios2_finalize()
+
+Previously we explored how to read using the input mode `adios2::Mode::Read`. Nonetheless, ADIOS has another input mode
+named `adios2::Mode::ReadRandomAccess`. `adios2::Mode::Read` mode allows data access only timestep by timestep using
+`BeginStep/EndStep`, but generally it is more memory efficient as ADIOS is only required to load metadata for the
+current timestep. `ReadRandomAccess` can only be used with file engines and involves loading all the file metadata at
+once. So it can be more memory intensive than `adios2::Mode::Read` mode, but allows reading data from any timestep using
+`SetStepSelection()`. If you use `adios2::Mode::ReadRandomAccess` mode, be sure to allocate enough memory to hold
+multiple steps of the variable content.  Note that ADIOS streaming
+engines (like SST, DataMan, etc.) do not support `ReadRandomAccess`
+mode.  Also newer file Engines like BP5 to not allow
+`BeginStep/EndStep` calls in `ReadRandomAccess` mode.
+
+.. code:: C++
+
+    ADIOS adios("config.xml", MPI_COMM_WORLD);
+    |
+    |   IO io = adios.DeclareIO(...);
+    |   |
+    |   |   Engine e = io.Open("InputFileName.bp", adios2::Mode::ReadRandomAccess);
+    |   |   |
+    |   |   |   Variable var = io.InquireVariable(...)
+    |   |   |   |   var.SetStepSelection()
+    |   |   |   |   e.Get(var, datapointer);
+    |   |   |   |
+    |   |   |
+    |   |   e.Close();
+    |   |
+    |   |--> IO goes out of scope
+    |
+    |--> ADIOS goes out of scope or adios2_finalize()
+
+

docs/user_guide/source/components/io.rst

@@ -196,7 +196,7 @@ A particular ``Engine`` type is set to the current ``IO`` component with the ``I
 Engine polymorphism is handled internally by the ``IO`` class, which allows subclassing future derived ``Engine`` types without changing the basic API.
 
 ``Engine`` objects are created in various modes.
-The available modes are ``adios2::Mode::Read``, ``adios2::Mode::Write``, ``adios2::Mode::Append``, ``adios2::Mode::Sync``, ``adios2::Mode::Deferred``, and ``adios2::Mode::Undefined``.
+The available modes are ``adios2::Mode::Read``, ``adios2::Mode::ReadRandomAccess``, ``adios2::Mode::Write``, ``adios2::Mode::Append``, ``adios2::Mode::Sync``, ``adios2::Mode::Deferred``, and ``adios2::Mode::Undefined``.
 
 
 .. code-block:: c++

docs/user_guide/source/engines/bp5.rst

@@ -130,6 +130,14 @@ This engine allows the user to fine tune the buffering operations through the fo
 
    #. **Threads**: Read side: Specify how many threads one process can use to speed up reading. The default value is *0*, to let the engine estimate the number of threads based on how many processes are running on the compute node and how many hardware threads are available on the compute node but it will use maximum 16 threads. Value *1* forces the engine to read everything within the main thread of the process. Other values specify the exact number of threads the engine can use. Although multithreaded reading works in a single *Get(adios2::Mode::Sync)* call if the read selection spans multiple data blocks in the file, the best parallelization is achieved by using deferred mode and reading everything in *PerformGets()/EndStep()*.   
 
+   #. **FlattenSteps**: This is a writer-side parameter specifies that the
+      reader should interpret multiple writer-created timesteps as a
+      single timestep, essentially flattening all Put()s into a single step.
+
+   #. **IgnoreFlattenSteps**: This is a reader-side parameter that
+      tells the reader to ignore any FlattenSteps parameter supplied
+      to the writer.
+
 ============================== ===================== ===========================================================
  **Key**                       **Value Format**      **Default** and Examples
 ============================== ===================== ===========================================================
@@ -156,6 +164,8 @@ This engine allows the user to fine tune the buffering operations through the fo
  StatsLevel                     integer, 0 or 1       **1**, 0
  MaxOpenFilesAtOnce             integer >= 0          **UINT_MAX**, 1024, 1
  Threads                        integer >= 0          **0**, 1, 32
+ FlattenSteps                   boolean               **off**, on, true, false
+ IgnoreFlattenSteps             boolean               **off**, on, true, false
 ============================== ===================== ===========================================================
 
 

docs/user_guide/source/setting_up/source/cmake.rst

@@ -8,7 +8,7 @@ To build ADIOS v2.x, clone the repository and invoke the canonical CMake build s
 
     $ git clone https://github.com/ornladios/ADIOS2.git ADIOS2
     $ mkdir adios2-build && cd adios2-build
-    $ cmake ../ADIOS2 cmake -DADIOS2_BUILD_EXAMPLES=ON
+    $ cmake ../ADIOS2 -DADIOS2_BUILD_EXAMPLES=ON
     -- The C compiler identification is GNU 9.4.0
     -- The CXX compiler identification is GNU 9.4.0
     ...
@@ -111,10 +111,11 @@ Optionally, run the tests (need to configure with ``-DBUILD_TESTING=ON`` cmake f
     Total Test time (real) =   95.95 sec
 
 
-And finally, use the standard invocation to install:
+And finally, use the standard invocation to install (setting the install path beforehand):
 
 .. code-block:: bash
 
+    $ cmake ../ADIOS2 -DCMAKE_INSTALL_PREFIX=/path/to/where/adios/will/be/installed
     $ make install
 
 

docs/user_guide/source/tutorials/helloWorld.rst

@@ -86,14 +86,15 @@ Start editing the skeleton file `ADIOS2/examples/hello/helloWorld/hello-world_tu
 
   adios2::Engine reader = io.Open("hello-world-cpp.bp", adios2::Mode::Read);
   std::string greeting;
+  reader.BeginStep();
   reader.Get(varGreeting, greeting);
   reader.EndStep();
   reader.Close();
   return greeting;
 
 .. note::
 
-  The ``BeginStep`` and ``EndStep`` calls are required when **reading** one step and multiple steps. We will see in
+  In Mode::Read, the ``BeginStep`` and ``EndStep`` calls are required when **reading** one step and multiple steps. We will see in
   another tutorial how to read multiple steps. It's important to note that the ``BeginStep`` should be called **before**
   all ``Inquire*`` / ``Available*`` function calls.
 

python/adios2/stream.py

@@ -43,6 +43,9 @@ def string_to_mode(mode: str) -> [bindings.Mode, bool]:
     elif mode == "rra":
         bmode = bindings.Mode.ReadRandomAccess
         read_mode = True
+    elif mode == "rfs":
+        bmode = bindings.Mode.ReadFlattenSteps
+        read_mode = True
     elif mode == "w":
         bmode = bindings.Mode.Write
     elif mode == "a":

source/adios2/core/VariableBase.cpp

@@ -221,6 +221,13 @@ void VariableBase::SetMemorySelection(const Box<Dims> &memorySelection)
     const Dims &memoryStart = memorySelection.first;
     const Dims &memoryCount = memorySelection.second;
 
+    if (memoryStart.empty() && memoryCount.empty())
+    {
+        m_MemoryStart.clear();
+        m_MemoryCount.clear();
+        return;
+    }
+
     if (m_SingleValue)
     {
         helper::Throw<std::invalid_argument>("Core", "VariableBase", "SetMemorySelection",

source/adios2/engine/bp5/BP5Engine.h

@@ -56,15 +56,33 @@ class BP5Engine
 
     format::BufferSTL m_MetadataIndex;
 
-    /** Positions of flags in Index Table Header that Reader uses */
-    static constexpr size_t m_IndexHeaderSize = 64;
-    static constexpr size_t m_EndianFlagPosition = 36;
-    static constexpr size_t m_BPVersionPosition = 37;
-    static constexpr size_t m_BPMinorVersionPosition = 38;
-    static constexpr size_t m_ActiveFlagPosition = 39;
-    static constexpr size_t m_ColumnMajorFlagPosition = 40;
-    static constexpr size_t m_VersionTagPosition = 0;
-    static constexpr size_t m_VersionTagLength = 32;
+    /** Positions of flags in Index Table Header that Reader uses - MUST BE 64 bytes total */
+    struct BP5IndexTableHeader
+    {
+        char VersionTag[32];
+        uint8_t adiosMajorVersion;
+        uint8_t adiosMinorVersion;
+        uint8_t adiosPatchVersion;
+        uint8_t unused1;        // init to zero
+        uint8_t isLittleEndian; // boolean
+        uint8_t bpVersion;      // 5 here
+        uint8_t bpMinorVersion;
+        uint8_t activeFlag;
+        char columnMajor;     // y or n
+        uint8_t flattenSteps; // writer requests all steps flattened to one on read
+        char unused2[22];     // init to zero
+    };
+    static constexpr size_t m_IndexHeaderSize = sizeof(BP5IndexTableHeader);
+    static constexpr size_t m_EndianFlagPosition = offsetof(BP5IndexTableHeader, isLittleEndian);
+    static constexpr size_t m_BPVersionPosition = offsetof(BP5IndexTableHeader, bpVersion);
+    static constexpr size_t m_BPMinorVersionPosition =
+        offsetof(BP5IndexTableHeader, bpMinorVersion);
+    static constexpr size_t m_ActiveFlagPosition = offsetof(BP5IndexTableHeader, activeFlag);
+    static constexpr size_t m_ColumnMajorFlagPosition = offsetof(BP5IndexTableHeader, columnMajor);
+    static constexpr size_t m_FlattenStepsPosition = offsetof(BP5IndexTableHeader, flattenSteps);
+    static constexpr size_t m_VersionTagPosition = offsetof(BP5IndexTableHeader, VersionTag);
+    static constexpr size_t m_VersionTagLength = sizeof(BP5IndexTableHeader().VersionTag);
+    static constexpr size_t m_HeaderTailPadding = sizeof(BP5IndexTableHeader().unused2);
 
     static constexpr uint8_t m_BP5MinorVersion = 2;
 
@@ -154,6 +172,8 @@ class BP5Engine
     MACRO(StatsBlockSize, SizeBytes, size_t, DefaultStatsBlockSize)                                \
     MACRO(Threads, UInt, unsigned int, 0)                                                          \
     MACRO(UseOneTimeAttributes, Bool, bool, true)                                                  \
+    MACRO(FlattenSteps, Bool, bool, false)                                                         \
+    MACRO(IgnoreFlattenSteps, Bool, bool, false)                                                   \
     MACRO(RemoteDataPath, String, std::string, "")                                                 \
     MACRO(MaxOpenFilesAtOnce, UInt, unsigned int, UINT_MAX)
 

source/adios2/engine/bp5/BP5Reader.cpp

@@ -72,7 +72,7 @@ void BP5Reader::InstallMetadataForTimestep(size_t Step)
         size_t ThisMDSize =
             helper::ReadValue<uint64_t>(m_Metadata.Data(), Position, m_Minifooter.IsLittleEndian);
         char *ThisMD = m_Metadata.Data() + MDPosition;
-        if (m_OpenMode == Mode::ReadRandomAccess)
+        if ((m_OpenMode == Mode::ReadRandomAccess) || (m_FlattenSteps))
         {
             m_BP5Deserializer->InstallMetaData(ThisMD, ThisMDSize, WriterRank, Step);
         }
@@ -98,7 +98,7 @@ StepStatus BP5Reader::BeginStep(StepMode mode, const float timeoutSeconds)
 {
     PERFSTUBS_SCOPED_TIMER("BP5Reader::BeginStep");
 
-    if (m_OpenMode == Mode::ReadRandomAccess)
+    if (m_OpenMode != Mode::Read)
     {
         helper::Throw<std::logic_error>("Engine", "BP5Reader", "BeginStep",
                                         "BeginStep called in random access mode");
@@ -184,7 +184,7 @@ size_t BP5Reader::CurrentStep() const { return m_CurrentStep; }
 
 void BP5Reader::EndStep()
 {
-    if (m_OpenMode == Mode::ReadRandomAccess)
+    if (m_OpenMode != Mode::Read)
     {
         helper::Throw<std::logic_error>("Engine", "BP5Reader", "EndStep",
                                         "EndStep called in random access mode");
@@ -802,8 +802,9 @@ void BP5Reader::UpdateBuffer(const TimePoint &timeoutInstant, const Seconds &pol
         // create the serializer object
         if (!m_BP5Deserializer)
         {
-            m_BP5Deserializer = new format::BP5Deserializer(m_WriterIsRowMajor, m_ReaderIsRowMajor,
-                                                            (m_OpenMode == Mode::ReadRandomAccess));
+            m_BP5Deserializer =
+                new format::BP5Deserializer(m_WriterIsRowMajor, m_ReaderIsRowMajor,
+                                            (m_OpenMode != Mode::Read), (m_FlattenSteps));
             m_BP5Deserializer->m_Engine = this;
         }
     }
@@ -900,7 +901,7 @@ void BP5Reader::UpdateBuffer(const TimePoint &timeoutInstant, const Seconds &pol
 
         m_Comm.Bcast(m_Metadata.Data(), inputSize, 0);
 
-        if (m_OpenMode == Mode::ReadRandomAccess)
+        if ((m_OpenMode == Mode::ReadRandomAccess) || m_FlattenSteps)
         {
             for (size_t Step = 0; Step < m_MetadataIndexTable.size(); Step++)
             {
@@ -977,6 +978,15 @@ size_t BP5Reader::ParseMetadataIndex(format::BufferSTL &bufferSTL, const size_t
         const uint8_t val =
             helper::ReadValue<uint8_t>(buffer, position, m_Minifooter.IsLittleEndian);
         m_WriterIsRowMajor = val == 'n';
+
+        position = m_FlattenStepsPosition;
+        const uint8_t flatten_val =
+            helper::ReadValue<uint8_t>(buffer, position, m_Minifooter.IsLittleEndian);
+        m_FlattenSteps = (flatten_val != 0);
+
+        if (m_Parameters.IgnoreFlattenSteps)
+            m_FlattenSteps = false;
+
         // move position to first row
         position = m_IndexHeaderSize;
     }
@@ -1306,7 +1316,13 @@ void BP5Reader::FlushProfiler()
     }
 }
 
-size_t BP5Reader::DoSteps() const { return m_StepsCount; }
+size_t BP5Reader::DoSteps() const
+{
+    if (m_FlattenSteps)
+        return 1;
+    else
+        return m_StepsCount;
+}
 
 void BP5Reader::NotifyEngineNoVarsQuery()
 {

source/adios2/engine/bp5/BP5Reader.h

@@ -57,6 +57,8 @@ class BP5Reader : public BP5Engine, public Engine
     MinVarInfo *MinBlocksInfo(const VariableBase &, const size_t Step) const;
     bool VarShape(const VariableBase &Var, const size_t Step, Dims &Shape) const;
     bool VariableMinMax(const VariableBase &, const size_t Step, MinMaxStruct &MinMax);
+    const char *VariableExprStr(const VariableBase &Var);
+    void SetFlattenMode(bool flatten) { m_FlattenSteps = flatten; };
 
 private:
     format::BP5Deserializer *m_BP5Deserializer = nullptr;
@@ -222,6 +224,7 @@ class BP5Reader : public BP5Engine, public Engine
     uint32_t m_WriterColumnMajor = 0;
     bool m_ReaderIsRowMajor = true;
     bool m_WriterIsRowMajor = true;
+    bool m_FlattenSteps = false; // set to true of writer requested all steps be flattened into 1
 
     format::BufferSTL m_MetadataIndex;
     format::BufferSTL m_MetaMetadata;

source/adios2/engine/bp5/BP5Writer.cpp

@@ -507,9 +507,11 @@ void BP5Writer::MarshalAttributes()
 #ifdef ADIOS2_HAVE_DERIVED_VARIABLE
 void BP5Writer::ComputeDerivedVariables()
 {
+    PERFSTUBS_SCOPED_TIMER("BP5Writer::ComputeDerivedVariables");
     auto const &m_VariablesDerived = m_IO.GetDerivedVariables();
     auto const &m_Variables = m_IO.GetVariables();
     // parse all derived variables
+    m_Profiler.Start("DeriveVars");
     for (auto it = m_VariablesDerived.begin(); it != m_VariablesDerived.end(); it++)
     {
         // identify the variables used in the derived variable
@@ -562,6 +564,7 @@ void BP5Writer::ComputeDerivedVariables()
             free(std::get<0>(derivedBlock));
         }
     }
+    m_Profiler.Stop("DeriveVars");
 }
 #endif
 
@@ -1218,9 +1221,12 @@ void BP5Writer::MakeHeader(std::vector<char> &buffer, size_t &position, const st
         helper::CopyToBuffer(buffer, position, version.c_str());
     };
 
-    // auto &buffer = b.m_Buffer;
-    // auto &position = b.m_Position;
-    // auto &absolutePosition = b.m_AbsolutePosition;
+    if (sizeof(BP5IndexTableHeader) != 64)
+    {
+        std::cerr << "BP6 Index Table Header must be 64 bytes" << std::endl;
+        exit(1);
+    }
+
     if (position > 0)
     {
         helper::Throw<std::invalid_argument>(
@@ -1273,11 +1279,7 @@ void BP5Writer::MakeHeader(std::vector<char> &buffer, size_t &position, const st
     lf_CopyVersionChar(majorVersion, buffer, position);
     lf_CopyVersionChar(minorVersion, buffer, position);
     lf_CopyVersionChar(patchVersion, buffer, position);
-    ++position;
-
-    // Note: Reader does process and use bytes 36-38 in
-    // BP4Deserialize.cpp::ParseMetadataIndex().
-    // Order and position must match there.
+    position = m_EndianFlagPosition;
 
     // byte 36: endianness
     if (position != m_EndianFlagPosition)
@@ -1327,8 +1329,9 @@ void BP5Writer::MakeHeader(std::vector<char> &buffer, size_t &position, const st
     const uint8_t columnMajor = (m_IO.m_ArrayOrder == ArrayOrdering::ColumnMajor) ? 'y' : 'n';
     helper::CopyToBuffer(buffer, position, &columnMajor);
 
-    // byte 41-63: unused
-    position += 23;
+    helper::CopyToBuffer(buffer, position, &m_Parameters.FlattenSteps);
+    // remainder  unused
+    position = m_IndexHeaderSize;
     // absolutePosition = position;
 }