External block storage for JSON/TOML backend via file system or S3

CMakeLists.txt

@@ -382,6 +382,8 @@ else()
 endif()
 unset(openPMD_REQUIRED_ADIOS2_COMPONENTS)
 
+find_package(AWSSDK REQUIRED COMPONENTS s3)
+
 # external library: pybind11 (optional)
 include(${openPMD_SOURCE_DIR}/cmake/dependencies/pybind11.cmake)
 
@@ -431,7 +433,12 @@ set(CORE_SOURCE
         src/snapshots/IteratorTraits.cpp
         src/snapshots/RandomAccessIterator.cpp
         src/snapshots/Snapshots.cpp
-        src/snapshots/StatefulIterator.cpp)
+        src/snapshots/StatefulIterator.cpp
+        src/toolkit/ExternalBlockStorage.cpp
+        src/toolkit/AwsBuilder.cpp
+        src/toolkit/Aws.cpp
+        src/toolkit/StdioBuilder.cpp
+        src/toolkit/Stdio.cpp)
 set(IO_SOURCE
         src/IO/AbstractIOHandler.cpp
         src/IO/AbstractIOHandlerImpl.cpp
@@ -551,6 +558,8 @@ if(openPMD_HAVE_ADIOS2)
     endif()
 endif()
 
+target_link_libraries(openPMD PUBLIC ${AWSSDK_LIBRARIES})
+
 # Runtime parameter and API status checks ("asserts")
 if(openPMD_USE_VERIFY)
     target_compile_definitions(openPMD PRIVATE openPMD_USE_VERIFY=1)

include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp

@@ -29,9 +29,11 @@
 #include "openPMD/auxiliary/JSON_internal.hpp"
 #include "openPMD/backend/Variant_internal.hpp"
 #include "openPMD/config.hpp"
+#include "openPMD/toolkit/ExternalBlockStorage.hpp"
 
 #include <istream>
 #include <nlohmann/json.hpp>
+#include <variant>
 #if openPMD_HAVE_MPI
 #include <mpi.h>
 #endif
@@ -153,8 +155,57 @@ void from_json(const nlohmann::json &j, std::complex<T> &p)
 }
 } // namespace std
 
+namespace openPMD::internal
+{
+auto jsonDatatypeToString(Datatype dt) -> std::string;
+
+struct JsonDatatypeHandling
+{
+    template <typename T>
+    static auto encodeDatatype(nlohmann::json &j) -> bool
+    {
+        auto const &needed_datatype =
+            jsonDatatypeToString(determineDatatype<T>());
+        if (auto it = j.find("datatype"); it != j.end())
+        {
+            return it.value().get<std::string>() == needed_datatype;
+        }
+        else
+        {
+            j["datatype"] = needed_datatype;
+            return true;
+        }
+    }
+
+    template <typename Functor, typename... Args>
+    static auto decodeDatatype(nlohmann::json const &j, Args &&...args) -> bool
+    {
+        if (auto it = j.find("datatype"); it != j.end())
+        {
+            switchDatasetType<Functor>(
+                stringToDatatype(it.value().get<std::string>()),
+                std::forward<Args>(args)...);
+            return true;
+        }
+        else
+        {
+            return false;
+        }
+    }
+};
+} // namespace openPMD::internal
+
 namespace openPMD
 {
+namespace dataset_mode_types
+{
+    struct Dataset_t
+    {};
+    struct Template_t
+    {};
+    using External_t = std::shared_ptr<ExternalBlockStorage>;
+} // namespace dataset_mode_types
+
 class JSONIOHandlerImpl : public AbstractIOHandlerImpl
 {
     using json = nlohmann::json;
@@ -243,42 +294,6 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl
 
     std::future<void> flush();
 
-private:
-#if openPMD_HAVE_MPI
-    std::optional<MPI_Comm> m_communicator;
-#endif
-
-    using FILEHANDLE = std::fstream;
-
-    // map each Writable to its associated file
-    // contains only the filename, without the OS path
-    std::unordered_map<Writable *, File> m_files;
-
-    std::unordered_map<File, std::shared_ptr<nlohmann::json>> m_jsonVals;
-
-    // files that have logically, but not physically been written to
-    std::unordered_set<File> m_dirty;
-
-    /*
-     * Is set by constructor.
-     */
-    FileFormat m_fileFormat{};
-
-    /*
-     * Under which key do we find the backend configuration?
-     * -> "json" for the JSON backend, "toml" for the TOML backend.
-     */
-    std::string backendConfigKey() const;
-
-    /*
-     * First return value: The location of the JSON value (either "json" or
-     * "toml") Second return value: The value that was maybe found at this place
-     */
-    std::pair<std::string, std::optional<openPMD::json::TracingJSON>>
-    getBackendConfig(openPMD::json::TracingJSON &) const;
-
-    std::string m_originalExtension;
-
     /*
      * Was the config value explicitly user-chosen, or are we still working with
      * defaults?
@@ -293,17 +308,36 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl
     // Dataset IO mode //
     /////////////////////
 
-    enum class DatasetMode
+    struct DatasetMode
+        : std::variant<
+              dataset_mode_types::Dataset_t,
+              dataset_mode_types::Template_t,
+              dataset_mode_types::External_t>
     {
-        Dataset,
-        Template
+        using Dataset_t = dataset_mode_types::Dataset_t;
+        using Template_t = dataset_mode_types::Template_t;
+        using External_t = dataset_mode_types::External_t;
+        constexpr static Dataset_t Dataset{};
+        constexpr static Template_t Template{};
+
+        using variant_t = std::variant<
+            dataset_mode_types::Dataset_t,
+            dataset_mode_types::Template_t,
+            External_t>;
+        using variant_t ::operator=;
+
+        // casts needed because of
+        // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90943
+        inline auto as_base() const -> variant_t const &
+        {
+            return *this;
+        }
+        inline auto as_base() -> variant_t &
+        {
+            return *this;
+        }
     };
 
-    // IOMode m_mode{};
-    // SpecificationVia m_IOModeSpecificationVia =
-    // SpecificationVia::DefaultValue; bool m_printedSkippedWriteWarningAlready
-    // = false;
-
     struct DatasetMode_s
     {
         // Initialized in init()
@@ -318,8 +352,6 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl
                 m_mode, m_specificationVia, m_skipWarnings};
         }
     };
-    DatasetMode_s m_datasetMode;
-    DatasetMode_s retrieveDatasetMode(openPMD::json::TracingJSON &config) const;
 
     ///////////////////////
     // Attribute IO mode //
@@ -338,8 +370,50 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl
         AttributeMode m_mode{};
         SpecificationVia m_specificationVia = SpecificationVia::DefaultValue;
     };
-    AttributeMode_s m_attributeMode;
 
+private:
+#if openPMD_HAVE_MPI
+    std::optional<MPI_Comm> m_communicator;
+#endif
+
+    using FILEHANDLE = std::fstream;
+
+    // map each Writable to its associated file
+    // contains only the filename, without the OS path
+    std::unordered_map<Writable *, File> m_files;
+
+    std::unordered_map<File, std::shared_ptr<nlohmann::json>> m_jsonVals;
+
+    // files that have logically, but not physically been written to
+    std::unordered_set<File> m_dirty;
+
+    /*
+     * Is set by constructor.
+     */
+    FileFormat m_fileFormat{};
+
+    /*
+     * Under which key do we find the backend configuration?
+     * -> "json" for the JSON backend, "toml" for the TOML backend.
+     */
+    std::string backendConfigKey() const;
+
+    /*
+     * First return value: The location of the JSON value (either "json" or
+     * "toml") Second return value: The value that was maybe found at this place
+     */
+    std::pair<std::string, std::optional<openPMD::json::TracingJSON>>
+    getBackendConfig(openPMD::json::TracingJSON &) const;
+    static std::pair<std::string, std::optional<openPMD::json::TracingJSON>>
+    getBackendConfig(
+        openPMD::json::TracingJSON &, std::string const &configLocation);
+
+    std::string m_originalExtension;
+
+    DatasetMode_s m_datasetMode;
+    DatasetMode_s retrieveDatasetMode(openPMD::json::TracingJSON &config) const;
+
+    AttributeMode_s m_attributeMode;
     AttributeMode_s
     retrieveAttributeMode(openPMD::json::TracingJSON &config) const;
 
@@ -389,7 +463,8 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl
     // essentially: m_i = \prod_{j=0}^{i-1} extent_j
     static Extent getMultiplicators(Extent const &extent);
 
-    static std::pair<Extent, DatasetMode> getExtent(nlohmann::json &j);
+    static std::pair<Extent, DatasetMode>
+    getExtent(nlohmann::json &j, DatasetMode const &baseMode);
 
     // remove single '/' in the beginning and end of a string
     static std::string removeSlashes(std::string);

include/openPMD/Series.hpp

@@ -35,6 +35,7 @@
 #include "openPMD/config.hpp"
 #include "openPMD/snapshots/Snapshots.hpp"
 #include "openPMD/version.hpp"
+#include <aws/core/Aws.h>
 
 #if openPMD_HAVE_MPI
 #include <mpi.h>
@@ -238,6 +239,8 @@ namespace internal
         std::optional<std::function<AbstractIOHandler *(Series &)>>
             m_deferred_initialization = std::nullopt;
 
+        std::optional<Aws::SDKOptions> m_manageAwsAPI = std::nullopt;
+
         void close();
 
 #if openPMD_HAVE_MPI

include/openPMD/toolkit/Aws.hpp

@@ -0,0 +1,27 @@
+#pragma once
+
+#include "openPMD/toolkit/ExternalBlockStorage.hpp"
+
+#include <aws/s3/S3Client.h>
+
+namespace openPMD::internal
+{
+struct ExternalBlockStorageAws : ExternalBlockStorageBackend
+{
+private:
+    Aws::S3::S3Client m_client;
+    std::string m_bucketName;
+    std::optional<std::string> m_endpoint;
+
+public:
+    ExternalBlockStorageAws(
+        Aws::S3::S3Client,
+        std::string bucketName,
+        std::optional<std::string> endpoint);
+    auto put(std::string const &identifier, void const *data, size_t len)
+        -> std::string override;
+    [[nodiscard]] auto externalStorageLocation() const
+        -> nlohmann::json override;
+    ~ExternalBlockStorageAws() override;
+};
+} // namespace openPMD::internal

include/openPMD/toolkit/AwsBuilder.hpp

@@ -0,0 +1,47 @@
+#pragma once
+
+#include <cstdint>
+#include <initializer_list>
+#include <optional>
+#include <string>
+
+namespace openPMD
+{
+class ExternalBlockStorage;
+}
+
+namespace openPMD::internal
+{
+struct AwsBuilder
+{
+    AwsBuilder(
+        std::string bucketName, std::string accessKeyId, std::string secretKey);
+
+    enum class Scheme : uint8_t
+    {
+        HTTP,
+        HTTPS
+    };
+    std::string m_bucketName;
+    std::string m_accessKeyId;
+    std::string m_secretKey;
+    std::optional<std::string> m_sessionToken;
+    std::initializer_list<std::string> m_credentials;
+    std::optional<std::string> m_endpointOverride;
+    std::optional<std::string> m_region;
+    std::optional<Scheme> m_scheme;
+    std::optional<bool> m_verifySSL;
+
+    auto setBucketName(std::string bucketName) -> AwsBuilder &;
+    auto setCredentials(std::string accessKeyId, std::string secretKey)
+        -> AwsBuilder &;
+    auto setSessionToken(std::string sessionToken) -> AwsBuilder &;
+    auto setEndpointOverride(std::string endpoint) -> AwsBuilder &;
+    auto setRegion(std::string regionName) -> AwsBuilder &;
+    auto setScheme(Scheme s) -> AwsBuilder &;
+    auto setVerifySSL(bool verify) -> AwsBuilder &;
+
+    operator ::openPMD::ExternalBlockStorage();
+    auto build() -> ::openPMD::ExternalBlockStorage;
+};
+} // namespace openPMD::internal