Skip to content

Commit 7d964c3

Browse files
franzpoeschelfranzpoeschel
committed
Automatically detect layout when reading
1 parent 3ef31f0 commit 7d964c3

File tree

3 files changed

+151
-109
lines changed

3 files changed

+151
-109
lines changed

‎include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp‎

Lines changed: 47 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -358,10 +358,15 @@ namespace ADIOS2Defaults
358358
constexpr const_str str_params = "parameters";
359359
constexpr const_str str_usesteps = "usesteps";
360360
constexpr const_str str_usesstepsAttribute = "__openPMD_internal/useSteps";
361-
} // namespace ADIOS2Defaults
361+
/*
362+
* 0: pre 2021-02-09
363+
* 1: 2021-02-09
364+
*/
365+
constexpr const_str str_layoutVersion = "__openPMD_internal/layoutVersion";
366+
} // namespace ADIOS2Defaults
362367

363-
namespace detail
364-
{
368+
namespace detail
369+
{
365370
// Helper structs for calls to the switchType function
366371

367372
struct DatasetReader
@@ -1302,41 +1307,52 @@ namespace detail
13021307
*/
13031308
bool finalized = false;
13041309

1305-
void
1306-
configure_IO( ADIOS2IOHandlerImpl & impl );
1307-
};
1310+
void configure_IO( ADIOS2IOHandlerImpl & impl );
13081311

1312+
uint32_t layoutVersion() const
1313+
{
1314+
switch( m_attributeLayout )
1315+
{
1316+
case AttributeLayout::ByAdiosAttributes:
1317+
return 0;
1318+
case AttributeLayout::ByAdiosVariables:
1319+
return 1;
1320+
}
1321+
throw std::runtime_error( "Unreachable!" );
1322+
}
1323+
};
13091324

1310-
} // namespace detail
1325+
} // namespace detail
13111326
#endif // openPMD_HAVE_ADIOS2
13121327

1313-
1314-
class ADIOS2IOHandler : public AbstractIOHandler
1315-
{
1328+
class ADIOS2IOHandler : public AbstractIOHandler
1329+
{
13161330
#if openPMD_HAVE_ADIOS2
13171331

1318-
friend class ADIOS2IOHandlerImpl;
1332+
friend class ADIOS2IOHandlerImpl;
13191333

1320-
private:
1321-
ADIOS2IOHandlerImpl m_impl;
1334+
private:
1335+
ADIOS2IOHandlerImpl m_impl;
13221336

1323-
public:
1324-
~ADIOS2IOHandler( ) override
1325-
{
1326-
// we must not throw in a destructor
1327-
try
1328-
{
1329-
this->flush( );
1330-
}
1331-
catch( std::exception const & ex )
1332-
{
1333-
std::cerr << "[~ADIOS2IOHandler] An error occurred: " << ex.what() << std::endl;
1334-
}
1335-
catch( ... )
1337+
public:
1338+
~ADIOS2IOHandler() override
13361339
{
1337-
std::cerr << "[~ADIOS2IOHandler] An error occurred." << std::endl;
1340+
// we must not throw in a destructor
1341+
try
1342+
{
1343+
this->flush();
1344+
}
1345+
catch( std::exception const & ex )
1346+
{
1347+
std::cerr << "[~ADIOS2IOHandler] An error occurred: "
1348+
<< ex.what() << std::endl;
1349+
}
1350+
catch( ... )
1351+
{
1352+
std::cerr << "[~ADIOS2IOHandler] An error occurred."
1353+
<< std::endl;
1354+
}
13381355
}
1339-
}
13401356

13411357
#else
13421358
public:
@@ -1361,6 +1377,6 @@ friend class ADIOS2IOHandlerImpl;
13611377

13621378
std::string backendName() const override { return "ADIOS2"; }
13631379

1364-
std::future< void > flush( ) override;
1365-
}; // ADIOS2IOHandler
1366-
} // namespace openPMD
1380+
std::future< void > flush() override;
1381+
}; // ADIOS2IOHandler
1382+
} // namespace openPMD

‎src/IO/ADIOS/ADIOS2IOHandler.cpp‎

Lines changed: 90 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -2154,6 +2154,7 @@ namespace detail
21542154
if( it != streamingEngines.end() )
21552155
{
21562156
isStreaming = true;
2157+
// @todo no?? should be default in both modes
21572158
optimizeAttributesStreaming =
21582159
m_attributeLayout == AttributeLayout::ByAdiosAttributes;
21592160
streamStatus = StreamStatus::OutsideOfStep;
@@ -2235,6 +2236,13 @@ namespace detail
22352236
throw std::runtime_error(
22362237
"Cannot switch off steps for streaming engines." );
22372238
}
2239+
if( m_attributeLayout == AttributeLayout::ByAdiosVariables &&
2240+
!bool( tmp ) )
2241+
{
2242+
throw std::runtime_error(
2243+
"[ADIOS2] Steps are mandatory in variable-based "
2244+
"attribute layout." );
2245+
}
22382246
streamStatus = bool( tmp ) ? StreamStatus::OutsideOfStep
22392247
: StreamStatus::NoStream;
22402248
}
@@ -2302,72 +2310,107 @@ namespace detail
23022310
*/
23032311
m_IO.SetParameter( "StatsLevel", "0" );
23042312
}
2313+
2314+
// We need to open the engine now already to inquire configuration
2315+
// options stored in there
2316+
getEngine();
23052317
}
23062318

2307-
adios2::Engine &
2308-
BufferedActions::getEngine()
2319+
adios2::Engine & BufferedActions::getEngine()
23092320
{
23102321
if( !m_engine )
23112322
{
23122323
switch( m_mode )
23132324
{
2314-
case adios2::Mode::Write:
2315-
{
2316-
bool_representation usesSteps =
2317-
streamStatus == StreamStatus::NoStream ? 0 : 1;
2318-
m_IO.DefineAttribute< bool_representation >(
2319-
ADIOS2Defaults::str_usesstepsAttribute, usesSteps );
2320-
m_engine = auxiliary::makeOption(
2321-
adios2::Engine( m_IO.Open( m_file, m_mode ) ) );
2322-
break;
2323-
}
2324-
case adios2::Mode::Read:
2325-
{
2326-
m_engine = auxiliary::makeOption(
2327-
adios2::Engine( m_IO.Open( m_file, m_mode ) ) );
2328-
switch( streamStatus )
2325+
case adios2::Mode::Write: {
2326+
bool_representation usesSteps =
2327+
streamStatus == StreamStatus::NoStream ? 0 : 1;
2328+
m_IO.DefineAttribute< bool_representation >(
2329+
ADIOS2Defaults::str_usesstepsAttribute, usesSteps );
2330+
m_IO.DefineAttribute< uint32_t >(
2331+
ADIOS2Defaults::str_layoutVersion, layoutVersion() );
2332+
m_engine = auxiliary::makeOption(
2333+
adios2::Engine( m_IO.Open( m_file, m_mode ) ) );
2334+
break;
2335+
}
2336+
case adios2::Mode::Read: {
2337+
m_engine = auxiliary::makeOption(
2338+
adios2::Engine( m_IO.Open( m_file, m_mode ) ) );
2339+
// decide attribute layout
2340+
// in streaming mode, this needs to be done after opening
2341+
// a step
2342+
// in file-based mode, we do it before
2343+
auto layoutVersion = [ IO{ m_IO } ]() mutable {
2344+
auto attr = IO.InquireAttribute< uint32_t >(
2345+
ADIOS2Defaults::str_layoutVersion );
2346+
if( !attr )
23292347
{
2330-
case StreamStatus::Undecided:
2348+
return AttributeLayout::ByAdiosAttributes;
2349+
}
2350+
else
2351+
{
2352+
switch( attr.Data()[ 0 ] )
23312353
{
2332-
auto attr =
2333-
m_IO.InquireAttribute< bool_representation >(
2334-
ADIOS2Defaults::str_usesstepsAttribute );
2335-
if( attr && attr.Data()[ 0 ] == 1 )
2336-
{
2337-
if( delayOpeningTheFirstStep )
2338-
{
2339-
streamStatus = StreamStatus::Parsing;
2340-
}
2341-
else
2342-
{
2343-
m_engine.get().BeginStep();
2344-
streamStatus = StreamStatus::DuringStep;
2345-
}
2346-
}
2347-
else
2348-
{
2349-
streamStatus = StreamStatus::NoStream;
2350-
}
2354+
case 0:
2355+
return AttributeLayout::ByAdiosAttributes;
23512356
break;
2352-
}
2353-
case StreamStatus::OutsideOfStep:
2354-
m_engine.get().BeginStep();
2355-
streamStatus = StreamStatus::DuringStep;
2357+
case 1:
2358+
return AttributeLayout::ByAdiosVariables;
23562359
break;
23572360
default:
23582361
throw std::runtime_error(
2359-
"[ADIOS2] Control flow error!" );
2362+
"[ADIOS2] Unknown semantic version for "
2363+
"data "
2364+
"layout: " +
2365+
std::to_string( attr.Data()[ 0 ] ) );
2366+
}
2367+
}
2368+
};
2369+
// decide streaming mode
2370+
switch( streamStatus )
2371+
{
2372+
case StreamStatus::Undecided: {
2373+
m_attributeLayout = layoutVersion();
2374+
auto attr = m_IO.InquireAttribute< bool_representation >(
2375+
ADIOS2Defaults::str_usesstepsAttribute );
2376+
// new attribute layout: steps are mandatory
2377+
// otherwise decide by flag
2378+
if( ( attr && attr.Data()[ 0 ] == 1 ) ||
2379+
m_attributeLayout == AttributeLayout::ByAdiosVariables )
2380+
{
2381+
if( delayOpeningTheFirstStep )
2382+
{
2383+
streamStatus = StreamStatus::Parsing;
2384+
}
2385+
else
2386+
{
2387+
m_engine.get().BeginStep();
2388+
streamStatus = StreamStatus::DuringStep;
2389+
}
23602390
}
2361-
if( m_attributeLayout == AttributeLayout::ByAdiosVariables )
2391+
else
23622392
{
2363-
preloadAttributes.preloadAttributes(
2364-
m_IO, m_engine.get() );
2393+
streamStatus = StreamStatus::NoStream;
23652394
}
23662395
break;
23672396
}
2397+
case StreamStatus::OutsideOfStep:
2398+
m_engine.get().BeginStep();
2399+
m_attributeLayout = layoutVersion();
2400+
streamStatus = StreamStatus::DuringStep;
2401+
break;
23682402
default:
2369-
throw std::runtime_error(
2370-
"[ADIOS2] Invalid ADIOS access mode" );
2403+
throw std::runtime_error( "[ADIOS2] Control flow error!" );
2404+
}
2405+
if( m_attributeLayout == AttributeLayout::ByAdiosVariables )
2406+
{
2407+
preloadAttributes.preloadAttributes( m_IO, m_engine.get() );
2408+
}
2409+
break;
2410+
}
2411+
default:
2412+
throw std::runtime_error(
2413+
"[ADIOS2] Invalid ADIOS access mode" );
23712414
}
23722415

23732416
if( !m_engine )

‎test/SerialIOTest.cpp‎

Lines changed: 14 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -3173,31 +3173,25 @@ TEST_CASE( "bp4_steps", "[serial][adios2]" )
31733173
}
31743174
}
31753175
)";
3176-
dontUseSteps = R"(
3177-
{
3178-
"adios2": {
3179-
"new_attribute_layout": true,
3180-
"engine": {
3181-
"type": "bp4",
3182-
"usesteps": false
3183-
}
3184-
}
3185-
}
3186-
)";
3176+
// steps are mandatory in new attribute layout
3177+
// dontUseSteps = R"(
3178+
// {
3179+
// "adios2": {
3180+
// "new_attribute_layout": true,
3181+
// "engine": {
3182+
// "type": "bp4",
3183+
// "usesteps": false
3184+
// }
3185+
// }
3186+
// }
3187+
// )";
31873188
/*
31883189
* @todo Activate these tests for Windows as soon as we bump the required
31893190
* ADIOS2 version to 2.7.0. Read here:
31903191
* https://github.com/openPMD/openPMD-api/pull/813#issuecomment-762235260
31913192
*/
31923193
#ifndef _WIN32
3193-
// sing the yes no song
31943194
bp4_steps( "../samples/newlayout_bp4steps_yes_yes.bp", useSteps, useSteps );
3195-
bp4_steps(
3196-
"../samples/newlayout_bp4steps_no_yes.bp", dontUseSteps, useSteps );
3197-
bp4_steps(
3198-
"../samples/newlayout_bp4steps_yes_no.bp", useSteps, dontUseSteps );
3199-
bp4_steps(
3200-
"../samples/newlayout_bp4steps_no_no.bp", dontUseSteps, dontUseSteps );
32013195
#endif
32023196
}
32033197
#endif
@@ -3294,20 +3288,9 @@ TEST_CASE( "stepBasedSingleIteration", "[serial][adios2]" )
32943288
void
32953289
stepBasedSeries( std::string const & file )
32963290
{
3297-
std::string config = R"END(
3298-
{
3299-
"adios2":
3300-
{
3301-
"new_attribute_layout": true,
3302-
"engine":
3303-
{
3304-
"usesteps": true
3305-
}
3306-
}
3307-
})END";
33083291
constexpr Extent::value_type extent = 1000;
33093292
{
3310-
Series writeSeries( file, Access::CREATE, config );
3293+
Series writeSeries( file, Access::CREATE );
33113294
if( writeSeries.backend() == "ADIOS1" )
33123295
{
33133296
return;
@@ -3326,7 +3309,7 @@ stepBasedSeries( std::string const & file )
33263309
}
33273310
}
33283311

3329-
Series readSeries( file, Access::READ_ONLY, config );
3312+
Series readSeries( file, Access::READ_ONLY );
33303313

33313314
size_t last_iteration_index = 0;
33323315
for( auto iteration : readSeries.readIterations() )

0 commit comments

Comments
 (0)